def _convert_query_histories(self): # Convert SQL Query history documents try: from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS docs = self._get_unconverted_docs(SavedQuery, only_history=True).filter(extra__in=[HQL, IMPALA, RDBMS]).order_by( '-last_modified') for doc in docs: try: if doc.content_object: notebook = import_saved_beeswax_query(doc.content_object) data = notebook.get_data() data['isSaved'] = False data['snippets'][0]['lastExecuted'] = time.mktime(doc.last_modified.timetuple()) * 1000 with transaction.atomic(): doc2 = _historify(data, self.user) doc2.last_modified = doc.last_modified # save() updates the last_modified to current time. Resetting it using update() doc2.save() Document2.objects.filter(id=doc2.id).update(last_modified=doc.last_modified) self.imported_doc_count += 1 doc.add_tag(self.imported_tag) doc.save() except Exception, e: self.failed_doc_ids.append(doc.id) LOG.exception('Failed to import history document id: %d' % doc.id) except ImportError, e: LOG.warn('Cannot convert history documents: beeswax app is not installed')
def test_historify(self): # Starts with no history assert_equal(0, Document2.objects.filter(name__contains=self.notebook['name'], is_history=True).count()) assert_equal(1, Document.objects.filter(name__contains=self.notebook['name']).count()) history_doc = _historify(self.notebook, self.user) assert_true(history_doc.id > 0) # Test that historify creates new Doc2 and linked Doc1 assert_equal(1, Document2.objects.filter(name__contains=self.notebook['name'], is_history=True).count()) assert_equal(2, Document.objects.filter(name__contains=self.notebook['name']).count()) # Historify again history_doc = _historify(self.notebook, self.user) assert_equal(2, Document2.objects.filter(name__contains=self.notebook['name'], is_history=True).count()) assert_equal(3, Document.objects.filter(name__contains=self.notebook['name']).count())
def test_clear_history(self): assert_equal( 0, Document2.objects.filter(name__contains=self.notebook['name'], is_history=True).count()) _historify(self.notebook, self.user) _historify(self.notebook, self.user) _historify(self.notebook, self.user) assert_equal( 3, Document2.objects.filter(name__contains=self.notebook['name'], is_history=True).count()) # Clear history should not clear history objects that don't have the given doc type Document2.objects.create(name='Impala History', type='query-impala', owner=self.user, is_history=True) # clear history should retain original document but wipe history response = self.client.post(reverse('notebook:clear_history'), { 'notebook': self.notebook_json, 'doc_type': 'hive' }) data = json.loads(response.content) assert_equal(0, data['status'], data) assert_false( Document2.objects.filter(type='query-hive', is_history=True).exists()) assert_true( Document2.objects.filter(type='query-hive', is_history=False).exists()) assert_true( Document2.objects.filter(type='query-impala', is_history=True).exists())
def test_get_history(self): assert_equal( 0, Document2.objects.filter(name__contains=self.notebook['name'], is_history=True).count()) _historify(self.notebook, self.user) _historify(self.notebook, self.user) _historify(self.notebook, self.user) assert_equal( 3, Document2.objects.filter(name__contains=self.notebook['name'], is_history=True).count()) # History should not return history objects that don't have the given doc type Document2.objects.create(name='Impala History', type='query-impala', data=self.notebook_json, owner=self.user, is_history=True) # Verify that get_history API returns history objects for given type and current user response = self.client.get(reverse('notebook:get_history'), {'doc_type': 'hive'}) data = json.loads(response.content) assert_equal(0, data['status'], data) assert_equal(3, len(data['history']), data) assert_true( all(doc['type'] == 'query-hive' for doc in data['history']), data)
def _convert_query_histories(self): # Convert SQL Query history documents try: from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS docs = self._get_unconverted_docs( SavedQuery, only_history=True).filter( extra__in=[HQL, IMPALA, RDBMS]).order_by('-last_modified') for doc in docs: try: if doc.content_object: notebook = import_saved_beeswax_query( doc.content_object) data = notebook.get_data() data['isSaved'] = False data['snippets'][0]['lastExecuted'] = time.mktime( doc.last_modified.timetuple()) * 1000 with transaction.atomic(): doc2 = _historify(data, self.user) doc2.last_modified = doc.last_modified # save() updates the last_modified to current time. Resetting it using update() doc2.save() Document2.objects.filter(id=doc2.id).update( last_modified=doc.last_modified) self.imported_doc_count += 1 doc.add_tag(self.imported_tag) doc.save() except Exception, e: self.failed_doc_ids.append(doc.id) LOG.exception('Failed to import history document id: %d' % doc.id) except ImportError, e: LOG.warn( 'Cannot convert history documents: beeswax app is not installed' )
def test_get_history(self): assert_equal(0, Document2.objects.filter(name__contains=self.notebook['name'], is_history=True).count()) _historify(self.notebook, self.user) _historify(self.notebook, self.user) _historify(self.notebook, self.user) assert_equal(3, Document2.objects.filter(name__contains=self.notebook['name'], is_history=True).count()) # History should not return history objects that don't have the given doc type Document2.objects.create(name='Impala History', type='query-impala', data=self.notebook_json, owner=self.user, is_history=True) # Verify that get_history API returns history objects for given type and current user response = self.client.get(reverse('notebook:get_history'), {'doc_type': 'hive'}) data = json.loads(response.content) assert_equal(0, data['status'], data) assert_equal(3, len(data['history']), data) assert_true(all(doc['type'] == 'query-hive' for doc in data['history']), data)
def test_clear_history(self): assert_equal(0, Document2.objects.filter(name__contains=self.notebook['name'], is_history=True).count()) _historify(self.notebook, self.user) _historify(self.notebook, self.user) _historify(self.notebook, self.user) assert_equal(3, Document2.objects.filter(name__contains=self.notebook['name'], is_history=True).count()) # Clear history should not clear history objects that don't have the given doc type Document2.objects.create(name='Impala History', type='query-impala', owner=self.user, is_history=True) # clear history should retain original document but wipe history response = self.client.post(reverse('notebook:clear_history'), {'notebook': self.notebook_json, 'doc_type': 'hive'}) data = json.loads(response.content) assert_equal(0, data['status'], data) assert_false(Document2.objects.filter(type='query-hive', is_history=True).exists()) assert_true(Document2.objects.filter(type='query-hive', is_history=False).exists()) assert_true(Document2.objects.filter(type='query-impala', is_history=True).exists())
def convert(self): # Convert SavedQuery documents try: from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS docs = self._get_unconverted_docs(SavedQuery).filter( extra__in=[HQL, IMPALA, RDBMS]) for doc in docs: if doc.content_object: notebook = import_saved_beeswax_query(doc.content_object) data = notebook.get_data() if doc.is_historic(): data['isSaved'] = False doc2 = self._create_doc2(document=doc, doctype=data['type'], name=data['name'], description=data['description'], data=notebook.get_json()) if doc.is_historic(): doc2.is_history = False self.imported_docs.append(doc2) except ImportError: LOG.warn( 'Cannot convert Saved Query documents: beeswax app is not installed' ) # Convert SQL Query history documents try: from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS docs = self._get_unconverted_docs( SavedQuery, with_history=True).filter( extra__in=[HQL, IMPALA, RDBMS]).order_by('-last_modified') for doc in docs: if doc.content_object: notebook = import_saved_beeswax_query(doc.content_object) data = notebook.get_data() data['isSaved'] = False data['snippets'][0]['lastExecuted'] = time.mktime( doc.last_modified.timetuple()) * 1000 doc2 = _historify(data, self.user) doc2.last_modified = doc.last_modified # save() updates the last_modified to current time. Resetting it using update() doc2.save() Document2.objects.filter(id=doc2.id).update( last_modified=doc.last_modified) self.imported_docs.append(doc2) # Tag for not re-importing Document.objects.link(doc2, owner=doc2.owner, name=doc2.name, description=doc2.description, extra=doc.extra) doc.add_tag(self.imported_tag) doc.save() except ImportError, e: LOG.warn( 'Cannot convert Saved Query documents: beeswax app is not installed' )
def convert(self): # Convert SavedQuery documents try: from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS docs = self._get_unconverted_docs(SavedQuery).filter(extra__in=[HQL, IMPALA, RDBMS]) for doc in docs: if doc.content_object: notebook = import_saved_beeswax_query(doc.content_object) data = notebook.get_data() if doc.is_historic(): data['isSaved'] = False doc2 = self._create_doc2( document=doc, doctype=data['type'], name=data['name'], description=data['description'], data=notebook.get_json() ) if doc.is_historic(): doc2.is_history = False self.imported_docs.append(doc2) except ImportError: LOG.warn('Cannot convert Saved Query documents: beeswax app is not installed') # Convert SQL Query history documents try: from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS docs = self._get_unconverted_docs(SavedQuery, with_history=True).filter(extra__in=[HQL, IMPALA, RDBMS]).order_by('-last_modified') for doc in docs: if doc.content_object: notebook = import_saved_beeswax_query(doc.content_object) data = notebook.get_data() data['isSaved'] = False data['snippets'][0]['lastExecuted'] = time.mktime(doc.last_modified.timetuple()) * 1000 doc2 = _historify(data, self.user) doc2.last_modified = doc.last_modified doc2.save() self.imported_docs.append(doc2) # Tag for not re-importing Document.objects.link( doc2, owner=doc2.owner, name=doc2.name, description=doc2.description, extra=doc.extra ) doc.add_tag(self.imported_tag) doc.save() except ImportError, e: LOG.warn('Cannot convert Saved Query documents: beeswax app is not installed')
class DocumentConverter(object): """ Given a user, converts any existing Document objects to Document2 objects """ def __init__(self, user): self.user = user # If user does not have a home directory, we need to create one and import any orphan documents to it self.home_dir = Document2.objects.create_user_directories(self.user) self.imported_tag = DocumentTag.objects.get_imported2_tag( user=self.user) self.imported_docs = [] def convert(self): # Convert SavedQuery documents try: from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS docs = self._get_unconverted_docs(SavedQuery).filter( extra__in=[HQL, IMPALA, RDBMS]) for doc in docs: if doc.content_object: notebook = import_saved_beeswax_query(doc.content_object) data = notebook.get_data() if doc.is_historic(): data['isSaved'] = False doc2 = self._create_doc2(document=doc, doctype=data['type'], name=data['name'], description=data['description'], data=notebook.get_json()) if doc.is_historic(): doc2.is_history = False self.imported_docs.append(doc2) except ImportError, e: LOG.warn( 'Cannot convert Saved Query documents: beeswax app is not installed' ) # Convert SQL Query history documents try: from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS docs = self._get_unconverted_docs( SavedQuery, with_history=True).filter( extra__in=[HQL, IMPALA, RDBMS]).order_by('-last_modified') for doc in docs: if doc.content_object: notebook = import_saved_beeswax_query(doc.content_object) data = notebook.get_data() data['isSaved'] = False data['snippets'][0]['lastExecuted'] = time.mktime( doc.last_modified.timetuple()) * 1000 doc2 = _historify(data, self.user) doc2.last_modified = doc.last_modified doc2.save() self.imported_docs.append(doc2) # Tag for not re-importing Document.objects.link(doc2, owner=doc2.owner, name=doc2.name, description=doc2.description, extra=doc.extra) doc.add_tag(self.imported_tag) doc.save() except ImportError, e: LOG.warn( 'Cannot convert Saved Query documents: beeswax app is not installed' )