def test_document_create(self):
sql = "SELECT * FROM sample_07"
design = hql_query(sql)
# is_auto
# is_trashed
# is_redacted
old_query = SavedQuery.objects.create(
type=SavedQuery.TYPES_MAPPING["hql"],
owner=self.user,
data=design.dumps(),
name="See examples",
desc="Example of old format",
)
try:
new_query = import_saved_beeswax_query(old_query)
new_query_data = new_query.get_data()
assert_equal("query-hive", new_query_data["type"])
assert_equal("See examples", new_query_data["name"])
assert_equal("Example of old format", new_query_data["description"])
assert_equal("ready", new_query_data["snippets"][0]["status"])
assert_equal("See examples", new_query_data["snippets"][0]["name"])
assert_equal("SELECT * FROM sample_07", new_query_data["snippets"][0]["statement_raw"])
assert_equal([], new_query_data["snippets"][0]["properties"]["settings"])
assert_equal([], new_query_data["snippets"][0]["properties"]["files"])
assert_equal([], new_query_data["snippets"][0]["properties"]["functions"])
finally:
old_query.delete()
def test_document_create(self):
sql = 'SELECT * FROM sample_07'
design = hql_query(sql)
# is_auto
# is_trashed
# is_redacted
old_query = SavedQuery.objects.create(
type=SavedQuery.TYPES_MAPPING['hql'],
owner=self.user,
data=design.dumps(),
name='See examples',
desc='Example of old format'
)
try:
new_query = import_saved_beeswax_query(old_query)
new_query_data = new_query.get_data()
assert_equal('query-hive', new_query_data['type'])
assert_equal('See examples', new_query_data['name'])
assert_equal('Example of old format', new_query_data['description'])
assert_equal('ready', new_query_data['snippets'][0]['status'])
assert_equal('See examples', new_query_data['snippets'][0]['name'])
assert_equal('SELECT * FROM sample_07', new_query_data['snippets'][0]['statement_raw'])
assert_equal([], new_query_data['snippets'][0]['properties']['settings'])
assert_equal([], new_query_data['snippets'][0]['properties']['files'])
assert_equal([], new_query_data['snippets'][0]['properties']['functions'])
finally:
old_query.delete()
def _import_documents1(user):
from beeswax.models import HQL, IMPALA, RDBMS
with transaction.atomic():
docs = Document.objects.get_docs(user, SavedQuery).filter(owner=user).filter(extra__in=[HQL, IMPALA, RDBMS])
imported_tag = DocumentTag.objects.get_imported2_tag(user=user)
docs = docs.exclude(tags__in=[
DocumentTag.objects.get_trash_tag(user=user), # No trashed docs
DocumentTag.objects.get_history_tag(user=user), # No history yet
DocumentTag.objects.get_example_tag(user=user), # No examples
imported_tag # No already imported docs
])
root_doc, created = Directory.objects.get_or_create(name='/', owner=user)
imported_docs = []
for doc in docs:
if doc.content_object:
try:
notebook = import_saved_beeswax_query(doc.content_object)
data = notebook.get_data()
notebook_doc = Document2.objects.create(name=data['name'], type=data['type'], owner=user, data=notebook.get_json())
doc.add_tag(imported_tag)
doc.save()
imported_docs.append(notebook_doc)
except Exception, e:
raise e
if imported_docs:
root_doc.dependencies.add(*imported_docs)
def test_document_create(self):
sql = 'SELECT * FROM sample_07'
design = hql_query(sql)
# is_auto
# is_trashed
# is_redacted
old_query = SavedQuery.objects.create(
type=SavedQuery.TYPES_MAPPING['hql'],
owner=self.user,
data=design.dumps(),
name='See examples',
desc='Example of old format'
)
try:
new_query = import_saved_beeswax_query(old_query)
new_query_data = new_query.get_data()
assert_equal('query-hive', new_query_data['type'])
assert_equal('See examples', new_query_data['name'])
assert_equal('Example of old format', new_query_data['description'])
assert_equal('ready', new_query_data['snippets'][0]['status'])
assert_equal('See examples', new_query_data['snippets'][0]['name'])
assert_equal('SELECT * FROM sample_07', new_query_data['snippets'][0]['statement_raw'])
assert_equal([], new_query_data['snippets'][0]['properties']['settings'])
assert_equal([], new_query_data['snippets'][0]['properties']['files'])
assert_equal([], new_query_data['snippets'][0]['properties']['functions'])
finally:
old_query.delete()
def _import_documents1(user):
from beeswax.models import HQL, IMPALA, RDBMS
docs = Document.objects.get_docs(user,
SavedQuery).filter(owner=user).filter(
extra__in=[HQL, IMPALA]) # TODO RDBMS
imported_tag = DocumentTag.objects.get_imported2_tag(user=user)
docs = docs.exclude(tags__in=[
DocumentTag.objects.get_trash_tag(user=user), # No trashed docs
DocumentTag.objects.get_history_tag(user=user), # No history yet
DocumentTag.objects.get_example_tag(user=user), # No examples
imported_tag # No already imported docs
])
root_doc, created = Directory.objects.get_or_create(name='/', owner=user)
imported_docs = []
for doc in docs:
if doc.content_object:
try:
notebook = import_saved_beeswax_query(doc.content_object)
data = notebook.get_data()
notebook_doc = Document2.objects.create(
name=data['name'],
type='query-%s' % data['type'],
owner=user,
data=notebook.get_json())
doc.add_tag(imported_tag)
doc.save()
imported_docs.append(notebook_doc)
except Exception, e:
raise e
def _import_documents1(user):
from beeswax.models import HQL, IMPALA, RDBMS
docs = Document.objects.get_docs(user, SavedQuery).filter(owner=user).filter(extra__in=[HQL, IMPALA]) # TODO RDBMS
imported_tag = DocumentTag.objects.get_imported2_tag(user=user)
docs = docs.exclude(
tags__in=[
DocumentTag.objects.get_trash_tag(user=user), # No trashed docs
DocumentTag.objects.get_history_tag(user=user), # No history yet
DocumentTag.objects.get_example_tag(user=user), # No examples
imported_tag, # No already imported docs
]
)
root_doc, created = Directory.objects.get_or_create(name="/", owner=user)
imported_docs = []
for doc in docs:
if doc.content_object:
try:
notebook = import_saved_beeswax_query(doc.content_object)
data = notebook.get_data()
notebook_doc = Document2.objects.create(
name=data["name"], type="query-%s" % data["type"], owner=user, data=notebook.get_json()
)
doc.add_tag(imported_tag)
doc.save()
imported_docs.append(notebook_doc)
except Exception, e:
raise e
def install(self, django_user):
"""
Install queries. Raise InstallException on failure.
"""
LOG.info('Installing sample query: %s' % (self.name, ))
try:
# Don't overwrite
query = SavedQuery.objects.get(owner=django_user,
name=self.name,
type=self.type)
except SavedQuery.DoesNotExist:
query = SavedQuery(owner=django_user,
name=self.name,
type=self.type,
desc=self.desc)
# The data field needs to be a string. The sample file writes it
# as json (without encoding into a string) for readability.
query.data = json.dumps(self.data)
query.save()
LOG.info('Successfully installed sample design: %s' %
(self.name, ))
if beeswax.conf.USE_NEW_EDITOR.get():
try:
# Don't overwrite
doc2 = Document2.objects.get(owner=django_user,
name=self.name,
type=self._document_type(
self.type))
except Document2.DoesNotExist:
# Create document from saved query
notebook = import_saved_beeswax_query(query)
data = notebook.get_json()
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(django_user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=django_user,
name=Document2.EXAMPLES_DIR)
doc2 = Document2.objects.create(owner=django_user,
parent_directory=examples_dir,
name=self.name,
type=self._document_type(
self.type),
description=self.desc,
data=data)
# Share with default group
examples_dir.share(django_user,
Document2Permission.READ_PERM,
groups=[get_default_user_group()])
doc2.save()
LOG.info('Successfully installed sample query: %s' %
(self.name, ))
def convert(self):
# Convert SavedQuery documents
docs = self._get_unconverted_docs(SavedQuery).filter(
extra__in=[HQL, IMPALA, RDBMS])
for doc in docs:
if doc.content_object:
notebook = import_saved_beeswax_query(doc.content_object)
data = notebook.get_data()
doc2 = self._create_doc2(document=doc,
doctype=data['type'],
name=data['name'],
description=data['description'],
data=notebook.get_json())
self.imported_docs.append(doc2)
# Convert Job Designer documents
# TODO: Change this logic to actually embed the workflow data in Doc2 instead of linking to old job design
docs = self._get_unconverted_docs(Workflow)
for doc in docs:
if doc.content_object:
data = doc.content_object.data_dict
data.update({
'content_type': doc.content_type.model,
'object_id': doc.object_id
})
doc2 = self._create_doc2(document=doc,
doctype='link-workflow',
description=doc.description,
data=json.dumps(data))
self.imported_docs.append(doc2)
# Convert PigScript documents
# TODO: Change this logic to actually embed the pig data in Doc2 instead of linking to old pig script
docs = self._get_unconverted_docs(PigScript)
for doc in docs:
if doc.content_object:
data = doc.content_object.dict
data.update({
'content_type': doc.content_type.model,
'object_id': doc.object_id
})
doc2 = self._create_doc2(document=doc,
doctype='link-pigscript',
description=doc.description,
data=json.dumps(data))
self.imported_docs.append(doc2)
# Add converted docs to root directory
if self.imported_docs:
LOG.info('Successfully imported %d documents' %
len(self.imported_docs))
def convert(self):
# Convert SavedQuery documents
docs = self._get_unconverted_docs(SavedQuery).filter(extra__in=[HQL, IMPALA, RDBMS])
for doc in docs:
if doc.content_object:
notebook = import_saved_beeswax_query(doc.content_object)
data = notebook.get_data()
doc2 = self._create_doc2(
document=doc,
doctype=data['type'],
name=data['name'],
description=data['description'],
data=notebook.get_json()
)
self.imported_docs.append(doc2)
# Convert Job Designer documents
# TODO: Change this logic to actually embed the workflow data in Doc2 instead of linking to old job design
docs = self._get_unconverted_docs(Workflow)
for doc in docs:
if doc.content_object:
data = doc.content_object.data_dict
data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
doc2 = self._create_doc2(
document=doc,
doctype='link-workflow',
description=doc.description,
data=json.dumps(data)
)
self.imported_docs.append(doc2)
# Convert PigScript documents
# TODO: Change this logic to actually embed the pig data in Doc2 instead of linking to old pig script
docs = self._get_unconverted_docs(PigScript)
for doc in docs:
if doc.content_object:
data = doc.content_object.dict
data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
doc2 = self._create_doc2(
document=doc,
doctype='link-pigscript',
description=doc.description,
data=json.dumps(data)
)
self.imported_docs.append(doc2)
# Add converted docs to root directory
if self.imported_docs:
LOG.info('Successfully imported %d documents' % len(self.imported_docs))
def install(self, django_user):
"""
Install queries. Raise InstallException on failure.
"""
LOG.info('Installing sample query: %s' % (self.name,))
try:
# Don't overwrite
query = SavedQuery.objects.get(owner=django_user, name=self.name, type=self.type)
except SavedQuery.DoesNotExist:
query = SavedQuery(owner=django_user, name=self.name, type=self.type, desc=self.desc)
# The data field needs to be a string. The sample file writes it
# as json (without encoding into a string) for readability.
query.data = json.dumps(self.data)
query.save()
LOG.info('Successfully installed sample design: %s' % (self.name,))
if beeswax.conf.USE_NEW_EDITOR.get():
try:
# Don't overwrite
doc2 = Document2.objects.get(owner=django_user, name=self.name, type=self._document_type(self.type))
except Document2.DoesNotExist:
# Create document from saved query
notebook = import_saved_beeswax_query(query)
data = notebook.get_json()
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(django_user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=django_user,
name=Document2.EXAMPLES_DIR
)
doc2 = Document2.objects.create(
owner=django_user,
parent_directory=examples_dir,
name=self.name,
type=self._document_type(self.type),
description=self.desc,
data=data
)
# Share with default group
examples_dir.share(django_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
doc2.save()
LOG.info('Successfully installed sample query: %s' % (self.name,))
def _convert_documents(user):
"""
Given a user, converts any existing Document objects to Document2 objects
"""
from beeswax.models import HQL, IMPALA, RDBMS
with transaction.atomic():
docs = Document.objects.get_docs(user, SavedQuery).filter(
owner=user).filter(extra__in=[HQL, IMPALA, RDBMS])
imported_tag = DocumentTag.objects.get_imported2_tag(user=user)
docs = docs.exclude(tags__in=[
DocumentTag.objects.get_trash_tag(user=user), # No trashed docs
DocumentTag.objects.get_history_tag(user=user), # No history yet
DocumentTag.objects.get_example_tag(user=user), # No examples
imported_tag # No already imported docs
])
root_doc, created = Directory.objects.get_or_create(name='/',
owner=user)
imported_docs = []
for doc in docs:
if doc.content_object:
try:
notebook = import_saved_beeswax_query(doc.content_object)
data = notebook.get_data()
notebook_doc = Document2.objects.create(
name=data['name'],
type=data['type'],
owner=user,
data=notebook.get_json())
doc.add_tag(imported_tag)
doc.save()
imported_docs.append(notebook_doc)
except Exception, e:
raise e
if imported_docs:
root_doc.dependencies.add(*imported_docs)
def convert(self):
# Convert SavedQuery documents
try:
from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
docs = self._get_unconverted_docs(SavedQuery).filter(extra__in=[HQL, IMPALA, RDBMS])
for doc in docs:
if doc.content_object:
notebook = import_saved_beeswax_query(doc.content_object)
data = notebook.get_data()
doc2 = self._create_doc2(
document=doc,
doctype=data['type'],
name=data['name'],
description=data['description'],
data=notebook.get_json()
)
self.imported_docs.append(doc2)
except ImportError, e:
LOG.warn('Cannot convert Saved Query documents: beeswax app is not installed')
def _convert_documents(user):
"""
Given a user, converts any existing Document objects to Document2 objects
"""
from beeswax.models import HQL, IMPALA, RDBMS
with transaction.atomic():
# If user does not have a home directory, we need to create one and import any orphan documents to it
Document2.objects.create_user_directories(user)
docs = Document.objects.get_docs(user, SavedQuery).filter(owner=user).filter(extra__in=[HQL, IMPALA, RDBMS])
imported_tag = DocumentTag.objects.get_imported2_tag(user=user)
docs = docs.exclude(tags__in=[
DocumentTag.objects.get_trash_tag(user=user), # No trashed docs
DocumentTag.objects.get_history_tag(user=user), # No history yet
DocumentTag.objects.get_example_tag(user=user), # No examples
imported_tag # No already imported docs
])
root_doc, created = Directory.objects.get_or_create(name='', owner=user)
imported_docs = []
for doc in docs:
if doc.content_object:
try:
notebook = import_saved_beeswax_query(doc.content_object)
data = notebook.get_data()
notebook_doc = Document2.objects.create(name=data['name'], type=data['type'], owner=user, data=notebook.get_json())
doc.add_tag(imported_tag)
doc.save()
imported_docs.append(notebook_doc)
except Exception, e:
raise e
if imported_docs:
root_doc.children.add(*imported_docs)