def _convert_saved_queries(self):
# Convert SavedQuery documents
try:
from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
docs = self._get_unconverted_docs(SavedQuery).filter(extra__in=[HQL, IMPALA, RDBMS])
for doc in docs:
try:
if doc.content_object:
notebook = import_saved_beeswax_query(doc.content_object)
data = notebook.get_data()
doc2 = self._create_doc2(
document=doc,
doctype=data['type'],
name=data['name'],
description=data['description'],
data=notebook.get_json()
)
self.imported_doc_count += 1
except Exception, e:
self.failed_doc_ids.append(doc.id)
LOG.exception('Failed to import SavedQuery document id: %d' % doc.id)
except ImportError:
LOG.warn('Cannot convert Saved Query documents: beeswax app is not installed')
def _convert_saved_queries(self):
# Convert SavedQuery documents
try:
from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
docs = self._get_unconverted_docs(SavedQuery).filter(
extra__in=[HQL, IMPALA, RDBMS])
for doc in docs:
try:
if doc.content_object:
notebook = import_saved_beeswax_query(
doc.content_object)
data = notebook.get_data()
doc2 = self._create_doc2(
document=doc,
doctype=data['type'],
name=data['name'],
description=data['description'],
data=notebook.get_json())
self.imported_doc_count += 1
except Exception, e:
self.failed_doc_ids.append(doc.id)
LOG.exception(
'Failed to import SavedQuery document id: %d' % doc.id)
except ImportError:
LOG.warn(
'Cannot convert Saved Query documents: beeswax app is not installed'
)
def test_document_create(self):
sql = 'SELECT * FROM sample_07'
design = hql_query(sql)
# is_auto
# is_trashed
# is_redacted
old_query = SavedQuery.objects.create(
type=SavedQuery.TYPES_MAPPING['hql'],
owner=self.user,
data=design.dumps(),
name='See examples',
desc='Example of old format'
)
try:
new_query = import_saved_beeswax_query(old_query)
new_query_data = new_query.get_data()
assert_equal('query-hive', new_query_data['type'])
assert_equal('See examples', new_query_data['name'])
assert_equal('Example of old format', new_query_data['description'])
assert_equal('ready', new_query_data['snippets'][0]['status'])
assert_equal('See examples', new_query_data['snippets'][0]['name'])
assert_equal('SELECT * FROM sample_07', new_query_data['snippets'][0]['statement_raw'])
assert_equal([], new_query_data['snippets'][0]['properties']['settings'])
assert_equal([], new_query_data['snippets'][0]['properties']['files'])
assert_equal([], new_query_data['snippets'][0]['properties']['functions'])
finally:
old_query.delete()
def test_document_create(self):
sql = 'SELECT * FROM sample_07'
design = hql_query(sql)
# is_auto
# is_trashed
# is_redacted
old_query = SavedQuery.objects.create(
type=SavedQuery.TYPES_MAPPING['hql'],
owner=self.user,
data=design.dumps(),
name='See examples',
desc='Example of old format'
)
try:
new_query = import_saved_beeswax_query(old_query)
new_query_data = new_query.get_data()
assert_equal('query-hive', new_query_data['type'])
assert_equal('See examples', new_query_data['name'])
assert_equal('Example of old format', new_query_data['description'])
assert_equal('ready', new_query_data['snippets'][0]['status'])
assert_equal('See examples', new_query_data['snippets'][0]['name'])
assert_equal('SELECT * FROM sample_07', new_query_data['snippets'][0]['statement_raw'])
assert_equal([], new_query_data['snippets'][0]['properties']['settings'])
assert_equal([], new_query_data['snippets'][0]['properties']['files'])
assert_equal([], new_query_data['snippets'][0]['properties']['functions'])
finally:
old_query.delete()
def convert(self):
# Convert SavedQuery documents
try:
from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
docs = self._get_unconverted_docs(SavedQuery).filter(extra__in=[HQL, IMPALA, RDBMS])
for doc in docs:
if doc.content_object:
notebook = import_saved_beeswax_query(doc.content_object)
data = notebook.get_data()
if doc.is_historic():
data['isSaved'] = False
doc2 = self._create_doc2(
document=doc,
doctype=data['type'],
name=data['name'],
description=data['description'],
data=notebook.get_json()
)
if doc.is_historic():
doc2.is_history = False
self.imported_docs.append(doc2)
except ImportError, e:
LOG.warn('Cannot convert Saved Query documents: beeswax app is not installed')
def _convert_query_histories(self):
# Convert SQL Query history documents
try:
from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
docs = self._get_unconverted_docs(SavedQuery, only_history=True).filter(extra__in=[HQL, IMPALA, RDBMS]).order_by(
'-last_modified')
for doc in docs:
try:
if doc.content_object:
notebook = import_saved_beeswax_query(doc.content_object)
data = notebook.get_data()
data['isSaved'] = False
data['snippets'][0]['lastExecuted'] = time.mktime(doc.last_modified.timetuple()) * 1000
with transaction.atomic():
doc2 = _historify(data, self.user)
doc2.last_modified = doc.last_modified
# save() updates the last_modified to current time. Resetting it using update()
doc2.save()
Document2.objects.filter(id=doc2.id).update(last_modified=doc.last_modified)
self.imported_doc_count += 1
doc.add_tag(self.imported_tag)
doc.save()
except Exception, e:
self.failed_doc_ids.append(doc.id)
LOG.exception('Failed to import history document id: %d' % doc.id)
except ImportError, e:
LOG.warn('Cannot convert history documents: beeswax app is not installed')
def convert(self):
# Convert SavedQuery documents
try:
from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
docs = self._get_unconverted_docs(SavedQuery).filter(
extra__in=[HQL, IMPALA, RDBMS])
for doc in docs:
if doc.content_object:
notebook = import_saved_beeswax_query(doc.content_object)
data = notebook.get_data()
if doc.is_historic():
data['isSaved'] = False
doc2 = self._create_doc2(document=doc,
doctype=data['type'],
name=data['name'],
description=data['description'],
data=notebook.get_json())
if doc.is_historic():
doc2.is_history = False
self.imported_docs.append(doc2)
except ImportError, e:
LOG.warn(
'Cannot convert Saved Query documents: beeswax app is not installed'
)
def install(self, django_user, interpreter=None):
"""
Install queries. Raise InstallException on failure.
"""
LOG.info('Installing sample query: %s' % (self.name,))
try:
# Don't overwrite
query = SavedQuery.objects.get(owner=django_user, name=self.name, type=self.type)
except SavedQuery.DoesNotExist:
query = SavedQuery(owner=django_user, name=self.name, type=self.type, desc=self.desc)
# The data field needs to be a string. The sample file writes it as json (without encoding into a string) for readability.
query.data = json.dumps(self.data)
query.save()
LOG.info('Successfully installed sample design: %s' % (self.name,))
if USE_NEW_EDITOR.get():
examples_dir = _get_example_directory(django_user)
document_type = self._document_type(self.type, interpreter)
notebook = import_saved_beeswax_query(query, interpreter=interpreter)
try:
# Could move PK from a uuid in queries.json at some point to handle name changes without duplicating.
# And move to a simpler json format at some point.
doc2 = Document2.objects.get(owner=django_user, name=self.name, type=document_type, is_history=False)
if doc2.parent_directory != examples_dir: # Recover from Trash or if moved
doc2.parent_directory = examples_dir
data = json.loads(doc2.data)
data['uuid'] = doc2.uuid
doc2.data = json.dumps(data) # Refresh content
doc2.save()
except Document2.DoesNotExist:
# Create new example
data = notebook.get_data()
data['isSaved'] = True
uuid = data.get('uuid')
data = json.dumps(data)
doc2 = Document2.objects.create(
uuid=uuid, # Must the same as in the notebook data
owner=django_user,
parent_directory=examples_dir,
name=self.name,
type=document_type,
description=self.desc,
data=data
)
# TODO: FK to connector object
# Share with default group
examples_dir.share(django_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
LOG.info('Successfully installed sample query: %s' % doc2)
def install(self, django_user, interpreter=None):
"""
Install queries. Raise InstallException on failure.
"""
LOG.info('Installing sample query: %s' % (self.name,))
try:
# Don't overwrite
query = SavedQuery.objects.get(owner=django_user, name=self.name, type=self.type)
except SavedQuery.DoesNotExist:
query = SavedQuery(owner=django_user, name=self.name, type=self.type, desc=self.desc)
# The data field needs to be a string. The sample file writes it
# as json (without encoding into a string) for readability.
query.data = json.dumps(self.data)
query.save()
LOG.info('Successfully installed sample design: %s' % (self.name,))
if USE_NEW_EDITOR.get():
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(django_user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=django_user,
name=Document2.EXAMPLES_DIR
)
document_type = self._document_type(self.type, interpreter)
try:
# Don't overwrite
doc2 = Document2.objects.get(owner=django_user, name=self.name, type=document_type, is_history=False)
# If document exists but has been trashed, recover from Trash
if doc2.parent_directory != examples_dir:
doc2.parent_directory = examples_dir
doc2.save()
except Document2.DoesNotExist:
# Create document from saved query
notebook = import_saved_beeswax_query(query, interpreter=interpreter)
data = notebook.get_data()
data['isSaved'] = True
uuid = data.get('uuid')
data = json.dumps(data)
doc2 = Document2.objects.create(
uuid=uuid,
owner=django_user,
parent_directory=examples_dir,
name=self.name,
type=document_type,
description=self.desc,
data=data
)
# Share with default group
examples_dir.share(django_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
LOG.info('Successfully installed sample query: %s' % (self.name,))
def install(self, django_user):
"""
Install queries. Raise InstallException on failure.
"""
LOG.info('Installing sample query: %s' % (self.name,))
try:
# Don't overwrite
query = SavedQuery.objects.get(owner=django_user, name=self.name, type=self.type)
except SavedQuery.DoesNotExist:
query = SavedQuery(owner=django_user, name=self.name, type=self.type, desc=self.desc)
# The data field needs to be a string. The sample file writes it
# as json (without encoding into a string) for readability.
query.data = json.dumps(self.data)
query.save()
LOG.info('Successfully installed sample design: %s' % (self.name,))
if USE_NEW_EDITOR.get():
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(django_user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=django_user,
name=Document2.EXAMPLES_DIR
)
try:
# Don't overwrite
doc2 = Document2.objects.get(owner=django_user, name=self.name, type=self._document_type(self.type))
# If document exists but has been trashed, recover from Trash
if doc2.parent_directory != examples_dir:
doc2.parent_directory = examples_dir
doc2.save()
except Document2.DoesNotExist:
# Create document from saved query
notebook = import_saved_beeswax_query(query)
data = notebook.get_data()
data['isSaved'] = True
uuid = data.get('uuid')
data = json.dumps(data)
doc2 = Document2.objects.create(
uuid=uuid,
owner=django_user,
parent_directory=examples_dir,
name=self.name,
type=self._document_type(self.type),
description=self.desc,
data=data
)
# Share with default group
examples_dir.share(django_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
LOG.info('Successfully installed sample query: %s' % (self.name,))
def _convert_query_histories(self):
# Convert SQL Query history documents
try:
from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
docs = self._get_unconverted_docs(
SavedQuery, only_history=True).filter(
extra__in=[HQL, IMPALA, RDBMS]).order_by('-last_modified')
for doc in docs:
try:
if doc.content_object:
notebook = import_saved_beeswax_query(
doc.content_object)
data = notebook.get_data()
data['isSaved'] = False
data['snippets'][0]['lastExecuted'] = time.mktime(
doc.last_modified.timetuple()) * 1000
with transaction.atomic():
doc2 = _historify(data, self.user)
doc2.last_modified = doc.last_modified
# save() updates the last_modified to current time. Resetting it using update()
doc2.save()
Document2.objects.filter(id=doc2.id).update(
last_modified=doc.last_modified)
self.imported_doc_count += 1
doc.add_tag(self.imported_tag)
doc.save()
except Exception, e:
self.failed_doc_ids.append(doc.id)
LOG.exception('Failed to import history document id: %d' %
doc.id)
except ImportError, e:
LOG.warn(
'Cannot convert history documents: beeswax app is not installed'
)
def convert(self):
# Convert SavedQuery documents
try:
from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
docs = self._get_unconverted_docs(SavedQuery).filter(
extra__in=[HQL, IMPALA, RDBMS])
for doc in docs:
if doc.content_object:
notebook = import_saved_beeswax_query(doc.content_object)
data = notebook.get_data()
if doc.is_historic():
data['isSaved'] = False
doc2 = self._create_doc2(document=doc,
doctype=data['type'],
name=data['name'],
description=data['description'],
data=notebook.get_json())
if doc.is_historic():
doc2.is_history = False
self.imported_docs.append(doc2)
except ImportError:
LOG.warn(
'Cannot convert Saved Query documents: beeswax app is not installed'
)
# Convert SQL Query history documents
try:
from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
docs = self._get_unconverted_docs(
SavedQuery, with_history=True).filter(
extra__in=[HQL, IMPALA, RDBMS]).order_by('-last_modified')
for doc in docs:
if doc.content_object:
notebook = import_saved_beeswax_query(doc.content_object)
data = notebook.get_data()
data['isSaved'] = False
data['snippets'][0]['lastExecuted'] = time.mktime(
doc.last_modified.timetuple()) * 1000
doc2 = _historify(data, self.user)
doc2.last_modified = doc.last_modified
# save() updates the last_modified to current time. Resetting it using update()
doc2.save()
Document2.objects.filter(id=doc2.id).update(
last_modified=doc.last_modified)
self.imported_docs.append(doc2)
# Tag for not re-importing
Document.objects.link(doc2,
owner=doc2.owner,
name=doc2.name,
description=doc2.description,
extra=doc.extra)
doc.add_tag(self.imported_tag)
doc.save()
except ImportError, e:
LOG.warn(
'Cannot convert Saved Query documents: beeswax app is not installed'
)
def convertfailed(self):
# Convert SavedQuery documents
try:
from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
docs = self._get_unconverted_docs(SavedQuery).filter(extra__in=[HQL, IMPALA, RDBMS])
for doc in docs:
if doc.content_object:
id_temp = doc.to_dict()
id = id_temp['id']
notebook = import_saved_beeswax_query(doc.content_object)
data = notebook.get_data()
name = data['name']
query = data['snippets'][0]['statement_raw']
if re.match(self.startqueryname, name) and not self.startuser:
self.processdocs = True
if self.processdocs:
matchdocs = findMatchingQuery(user=self.user, id=id, name=name, query=query, include_history=False)
if not matchdocs or self.allowdupes:
try:
if doc.is_historic():
data['isSaved'] = False
doc2 = self._create_doc2(
document=doc,
doctype=data['type'],
name=data['name'],
description=data['description'],
data=notebook.get_json()
)
if doc.is_historic():
doc2.is_history = False
self.imported_docs.append(doc2)
except:
pass
except ImportError:
LOG.info('Cannot convert Saved Query documents: beeswax app is not installed')
pass
# Convert SQL Query history documents
try:
from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
docs = self._get_unconverted_docs(SavedQuery, with_history=True).filter(extra__in=[HQL, IMPALA, RDBMS]).order_by('-last_modified')
for doc in docs:
if not doc.content_object:
LOG.error("Content object is missing")
elif doc.content_object:
id_temp = doc.to_dict()
id = id_temp['id']
notebook = import_saved_beeswax_query(doc.content_object)
data = notebook.get_data()
name = data['name']
query = data['snippets'][0]['statement_raw']
if re.match(self.startqueryname, name) and not self.startuser:
self.processdocs = True
if self.processdocs:
try:
data['isSaved'] = False
data['snippets'][0]['lastExecuted'] = time.mktime(doc.last_modified.timetuple()) * 1000
doc2 = self._historify(data, self.user)
doc2.last_modified = doc.last_modified
# save() updates the last_modified to current time. Resetting it using update()
doc2.save()
Document2.objects.filter(id=doc2.id).update(last_modified=doc.last_modified)
self.imported_docs.append(doc2)
# Tag for not re-importing
Document.objects.link(
doc2,
owner=doc2.owner,
name=doc2.name,
description=doc2.description,
extra=doc.extra
)
try:
doc.add_tag(self.imported_tag)
except IntegrityError, e:
LOG.exception("Failed to add imported_tag to doc %s with error %s" % (doc2.name, e))
pass
doc.save()
except:
def convert(self):
# Convert SavedQuery documents
try:
from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
docs = self._get_unconverted_docs(SavedQuery).filter(extra__in=[HQL, IMPALA, RDBMS])
for doc in docs:
if doc.content_object:
notebook = import_saved_beeswax_query(doc.content_object)
data = notebook.get_data()
if doc.is_historic():
data['isSaved'] = False
doc2 = self._create_doc2(
document=doc,
doctype=data['type'],
name=data['name'],
description=data['description'],
data=notebook.get_json()
)
if doc.is_historic():
doc2.is_history = False
self.imported_docs.append(doc2)
except ImportError:
LOG.warn('Cannot convert Saved Query documents: beeswax app is not installed')
# Convert SQL Query history documents
try:
from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
docs = self._get_unconverted_docs(SavedQuery, with_history=True).filter(extra__in=[HQL, IMPALA, RDBMS]).order_by('-last_modified')
for doc in docs:
if doc.content_object:
notebook = import_saved_beeswax_query(doc.content_object)
data = notebook.get_data()
data['isSaved'] = False
data['snippets'][0]['lastExecuted'] = time.mktime(doc.last_modified.timetuple()) * 1000
doc2 = _historify(data, self.user)
doc2.last_modified = doc.last_modified
doc2.save()
self.imported_docs.append(doc2)
# Tag for not re-importing
Document.objects.link(
doc2,
owner=doc2.owner,
name=doc2.name,
description=doc2.description,
extra=doc.extra
)
doc.add_tag(self.imported_tag)
doc.save()
except ImportError, e:
LOG.warn('Cannot convert Saved Query documents: beeswax app is not installed')
class DocumentConverter(object):
"""
Given a user, converts any existing Document objects to Document2 objects
"""
def __init__(self, user):
self.user = user
# If user does not have a home directory, we need to create one and import any orphan documents to it
self.home_dir = Document2.objects.create_user_directories(self.user)
self.imported_tag = DocumentTag.objects.get_imported2_tag(
user=self.user)
self.imported_docs = []
def convert(self):
# Convert SavedQuery documents
try:
from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
docs = self._get_unconverted_docs(SavedQuery).filter(
extra__in=[HQL, IMPALA, RDBMS])
for doc in docs:
if doc.content_object:
notebook = import_saved_beeswax_query(doc.content_object)
data = notebook.get_data()
if doc.is_historic():
data['isSaved'] = False
doc2 = self._create_doc2(document=doc,
doctype=data['type'],
name=data['name'],
description=data['description'],
data=notebook.get_json())
if doc.is_historic():
doc2.is_history = False
self.imported_docs.append(doc2)
except ImportError, e:
LOG.warn(
'Cannot convert Saved Query documents: beeswax app is not installed'
)
# Convert SQL Query history documents
try:
from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
docs = self._get_unconverted_docs(
SavedQuery, with_history=True).filter(
extra__in=[HQL, IMPALA, RDBMS]).order_by('-last_modified')
for doc in docs:
if doc.content_object:
notebook = import_saved_beeswax_query(doc.content_object)
data = notebook.get_data()
data['isSaved'] = False
data['snippets'][0]['lastExecuted'] = time.mktime(
doc.last_modified.timetuple()) * 1000
doc2 = _historify(data, self.user)
doc2.last_modified = doc.last_modified
doc2.save()
self.imported_docs.append(doc2)
# Tag for not re-importing
Document.objects.link(doc2,
owner=doc2.owner,
name=doc2.name,
description=doc2.description,
extra=doc.extra)
doc.add_tag(self.imported_tag)
doc.save()
except ImportError, e:
LOG.warn(
'Cannot convert Saved Query documents: beeswax app is not installed'
)
