Example #1
0
  def _convert_query_histories(self):
    # Convert SQL Query history documents
    try:
      from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS

      docs = self._get_unconverted_docs(SavedQuery, only_history=True).filter(extra__in=[HQL, IMPALA, RDBMS]).order_by(
        '-last_modified')

      for doc in docs:
        try:
          if doc.content_object:
            notebook = import_saved_beeswax_query(doc.content_object)
            data = notebook.get_data()

            data['isSaved'] = False
            data['snippets'][0]['lastExecuted'] = time.mktime(doc.last_modified.timetuple()) * 1000

            with transaction.atomic():
              doc2 = _historify(data, self.user)
              doc2.last_modified = doc.last_modified

              # save() updates the last_modified to current time. Resetting it using update()
              doc2.save()
              Document2.objects.filter(id=doc2.id).update(last_modified=doc.last_modified)

              self.imported_doc_count += 1

              doc.add_tag(self.imported_tag)
              doc.save()
        except Exception, e:
          self.failed_doc_ids.append(doc.id)
          LOG.exception('Failed to import history document id: %d' % doc.id)
    except ImportError, e:
      LOG.warn('Cannot convert history documents: beeswax app is not installed')
Example #2
0
  def test_historify(self):
    # Starts with no history
    assert_equal(0, Document2.objects.filter(name__contains=self.notebook['name'], is_history=True).count())
    assert_equal(1, Document.objects.filter(name__contains=self.notebook['name']).count())

    history_doc = _historify(self.notebook, self.user)

    assert_true(history_doc.id > 0)

    # Test that historify creates new Doc2 and linked Doc1
    assert_equal(1, Document2.objects.filter(name__contains=self.notebook['name'], is_history=True).count())
    assert_equal(2, Document.objects.filter(name__contains=self.notebook['name']).count())

    # Historify again
    history_doc = _historify(self.notebook, self.user)

    assert_equal(2, Document2.objects.filter(name__contains=self.notebook['name'], is_history=True).count())
    assert_equal(3, Document.objects.filter(name__contains=self.notebook['name']).count())
Example #3
0
  def test_historify(self):
    # Starts with no history
    assert_equal(0, Document2.objects.filter(name__contains=self.notebook['name'], is_history=True).count())
    assert_equal(1, Document.objects.filter(name__contains=self.notebook['name']).count())

    history_doc = _historify(self.notebook, self.user)

    assert_true(history_doc.id > 0)

    # Test that historify creates new Doc2 and linked Doc1
    assert_equal(1, Document2.objects.filter(name__contains=self.notebook['name'], is_history=True).count())
    assert_equal(2, Document.objects.filter(name__contains=self.notebook['name']).count())

    # Historify again
    history_doc = _historify(self.notebook, self.user)

    assert_equal(2, Document2.objects.filter(name__contains=self.notebook['name'], is_history=True).count())
    assert_equal(3, Document.objects.filter(name__contains=self.notebook['name']).count())
Example #4
0
File: tests.py Project: hombin/hue
    def test_clear_history(self):
        assert_equal(
            0,
            Document2.objects.filter(name__contains=self.notebook['name'],
                                     is_history=True).count())
        _historify(self.notebook, self.user)
        _historify(self.notebook, self.user)
        _historify(self.notebook, self.user)
        assert_equal(
            3,
            Document2.objects.filter(name__contains=self.notebook['name'],
                                     is_history=True).count())

        # Clear history should not clear history objects that don't have the given doc type
        Document2.objects.create(name='Impala History',
                                 type='query-impala',
                                 owner=self.user,
                                 is_history=True)

        # clear history should retain original document but wipe history
        response = self.client.post(reverse('notebook:clear_history'), {
            'notebook': self.notebook_json,
            'doc_type': 'hive'
        })
        data = json.loads(response.content)
        assert_equal(0, data['status'], data)
        assert_false(
            Document2.objects.filter(type='query-hive',
                                     is_history=True).exists())
        assert_true(
            Document2.objects.filter(type='query-hive',
                                     is_history=False).exists())
        assert_true(
            Document2.objects.filter(type='query-impala',
                                     is_history=True).exists())
Example #5
0
File: tests.py Project: hombin/hue
    def test_get_history(self):
        assert_equal(
            0,
            Document2.objects.filter(name__contains=self.notebook['name'],
                                     is_history=True).count())
        _historify(self.notebook, self.user)
        _historify(self.notebook, self.user)
        _historify(self.notebook, self.user)
        assert_equal(
            3,
            Document2.objects.filter(name__contains=self.notebook['name'],
                                     is_history=True).count())

        # History should not return history objects that don't have the given doc type
        Document2.objects.create(name='Impala History',
                                 type='query-impala',
                                 data=self.notebook_json,
                                 owner=self.user,
                                 is_history=True)

        # Verify that get_history API returns history objects for given type and current user
        response = self.client.get(reverse('notebook:get_history'),
                                   {'doc_type': 'hive'})
        data = json.loads(response.content)
        assert_equal(0, data['status'], data)
        assert_equal(3, len(data['history']), data)
        assert_true(
            all(doc['type'] == 'query-hive' for doc in data['history']), data)
Example #6
0
    def _convert_query_histories(self):
        # Convert SQL Query history documents
        try:
            from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS

            docs = self._get_unconverted_docs(
                SavedQuery, only_history=True).filter(
                    extra__in=[HQL, IMPALA, RDBMS]).order_by('-last_modified')

            for doc in docs:
                try:
                    if doc.content_object:
                        notebook = import_saved_beeswax_query(
                            doc.content_object)
                        data = notebook.get_data()

                        data['isSaved'] = False
                        data['snippets'][0]['lastExecuted'] = time.mktime(
                            doc.last_modified.timetuple()) * 1000

                        with transaction.atomic():
                            doc2 = _historify(data, self.user)
                            doc2.last_modified = doc.last_modified

                            # save() updates the last_modified to current time. Resetting it using update()
                            doc2.save()
                            Document2.objects.filter(id=doc2.id).update(
                                last_modified=doc.last_modified)

                            self.imported_doc_count += 1

                            doc.add_tag(self.imported_tag)
                            doc.save()
                except Exception, e:
                    self.failed_doc_ids.append(doc.id)
                    LOG.exception('Failed to import history document id: %d' %
                                  doc.id)
        except ImportError, e:
            LOG.warn(
                'Cannot convert history documents: beeswax app is not installed'
            )
Example #7
0
  def test_get_history(self):
    assert_equal(0, Document2.objects.filter(name__contains=self.notebook['name'], is_history=True).count())
    _historify(self.notebook, self.user)
    _historify(self.notebook, self.user)
    _historify(self.notebook, self.user)
    assert_equal(3, Document2.objects.filter(name__contains=self.notebook['name'], is_history=True).count())

    # History should not return history objects that don't have the given doc type
    Document2.objects.create(name='Impala History', type='query-impala', data=self.notebook_json, owner=self.user, is_history=True)

    # Verify that get_history API returns history objects for given type and current user
    response = self.client.get(reverse('notebook:get_history'), {'doc_type': 'hive'})
    data = json.loads(response.content)
    assert_equal(0, data['status'], data)
    assert_equal(3, len(data['history']), data)
    assert_true(all(doc['type'] == 'query-hive' for doc in data['history']), data)
Example #8
0
  def test_clear_history(self):
    assert_equal(0, Document2.objects.filter(name__contains=self.notebook['name'], is_history=True).count())
    _historify(self.notebook, self.user)
    _historify(self.notebook, self.user)
    _historify(self.notebook, self.user)
    assert_equal(3, Document2.objects.filter(name__contains=self.notebook['name'], is_history=True).count())

    # Clear history should not clear history objects that don't have the given doc type
    Document2.objects.create(name='Impala History', type='query-impala', owner=self.user, is_history=True)

    # clear history should retain original document but wipe history
    response = self.client.post(reverse('notebook:clear_history'), {'notebook': self.notebook_json, 'doc_type': 'hive'})
    data = json.loads(response.content)
    assert_equal(0, data['status'], data)
    assert_false(Document2.objects.filter(type='query-hive', is_history=True).exists())
    assert_true(Document2.objects.filter(type='query-hive', is_history=False).exists())
    assert_true(Document2.objects.filter(type='query-impala', is_history=True).exists())
Example #9
0
    def convert(self):
        # Convert SavedQuery documents
        try:
            from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS

            docs = self._get_unconverted_docs(SavedQuery).filter(
                extra__in=[HQL, IMPALA, RDBMS])
            for doc in docs:
                if doc.content_object:
                    notebook = import_saved_beeswax_query(doc.content_object)
                    data = notebook.get_data()

                    if doc.is_historic():
                        data['isSaved'] = False

                    doc2 = self._create_doc2(document=doc,
                                             doctype=data['type'],
                                             name=data['name'],
                                             description=data['description'],
                                             data=notebook.get_json())

                    if doc.is_historic():
                        doc2.is_history = False

                    self.imported_docs.append(doc2)
        except ImportError:
            LOG.warn(
                'Cannot convert Saved Query documents: beeswax app is not installed'
            )

        # Convert SQL Query history documents
        try:
            from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS

            docs = self._get_unconverted_docs(
                SavedQuery, with_history=True).filter(
                    extra__in=[HQL, IMPALA, RDBMS]).order_by('-last_modified')

            for doc in docs:
                if doc.content_object:
                    notebook = import_saved_beeswax_query(doc.content_object)
                    data = notebook.get_data()

                    data['isSaved'] = False
                    data['snippets'][0]['lastExecuted'] = time.mktime(
                        doc.last_modified.timetuple()) * 1000

                    doc2 = _historify(data, self.user)
                    doc2.last_modified = doc.last_modified

                    # save() updates the last_modified to current time. Resetting it using update()
                    doc2.save()
                    Document2.objects.filter(id=doc2.id).update(
                        last_modified=doc.last_modified)

                    self.imported_docs.append(doc2)

                    # Tag for not re-importing
                    Document.objects.link(doc2,
                                          owner=doc2.owner,
                                          name=doc2.name,
                                          description=doc2.description,
                                          extra=doc.extra)

                    doc.add_tag(self.imported_tag)
                    doc.save()
        except ImportError, e:
            LOG.warn(
                'Cannot convert Saved Query documents: beeswax app is not installed'
            )
Example #10
0
  def convert(self):
    # Convert SavedQuery documents
    try:
      from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS

      docs = self._get_unconverted_docs(SavedQuery).filter(extra__in=[HQL, IMPALA, RDBMS])
      for doc in docs:
        if doc.content_object:
          notebook = import_saved_beeswax_query(doc.content_object)
          data = notebook.get_data()

          if doc.is_historic():
            data['isSaved'] = False

          doc2 = self._create_doc2(
              document=doc,
              doctype=data['type'],
              name=data['name'],
              description=data['description'],
              data=notebook.get_json()
          )

          if doc.is_historic():
            doc2.is_history = False

          self.imported_docs.append(doc2)
    except ImportError:
      LOG.warn('Cannot convert Saved Query documents: beeswax app is not installed')

    # Convert SQL Query history documents
    try:
      from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS

      docs = self._get_unconverted_docs(SavedQuery, with_history=True).filter(extra__in=[HQL, IMPALA, RDBMS]).order_by('-last_modified')

      for doc in docs:
        if doc.content_object:
          notebook = import_saved_beeswax_query(doc.content_object)
          data = notebook.get_data()

          data['isSaved'] = False
          data['snippets'][0]['lastExecuted'] = time.mktime(doc.last_modified.timetuple()) * 1000

          doc2 = _historify(data, self.user)
          doc2.last_modified = doc.last_modified
          doc2.save()

          self.imported_docs.append(doc2)

          # Tag for not re-importing
          Document.objects.link(
            doc2,
            owner=doc2.owner,
            name=doc2.name,
            description=doc2.description,
            extra=doc.extra
          )

          doc.add_tag(self.imported_tag)
          doc.save()
    except ImportError, e:
      LOG.warn('Cannot convert Saved Query documents: beeswax app is not installed')
Example #11
0
class DocumentConverter(object):
    """
  Given a user, converts any existing Document objects to Document2 objects
  """
    def __init__(self, user):
        self.user = user
        # If user does not have a home directory, we need to create one and import any orphan documents to it
        self.home_dir = Document2.objects.create_user_directories(self.user)
        self.imported_tag = DocumentTag.objects.get_imported2_tag(
            user=self.user)
        self.imported_docs = []

    def convert(self):
        # Convert SavedQuery documents
        try:
            from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS

            docs = self._get_unconverted_docs(SavedQuery).filter(
                extra__in=[HQL, IMPALA, RDBMS])
            for doc in docs:
                if doc.content_object:
                    notebook = import_saved_beeswax_query(doc.content_object)
                    data = notebook.get_data()

                    if doc.is_historic():
                        data['isSaved'] = False

                    doc2 = self._create_doc2(document=doc,
                                             doctype=data['type'],
                                             name=data['name'],
                                             description=data['description'],
                                             data=notebook.get_json())

                    if doc.is_historic():
                        doc2.is_history = False

                    self.imported_docs.append(doc2)
        except ImportError, e:
            LOG.warn(
                'Cannot convert Saved Query documents: beeswax app is not installed'
            )

        # Convert SQL Query history documents
        try:
            from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS

            docs = self._get_unconverted_docs(
                SavedQuery, with_history=True).filter(
                    extra__in=[HQL, IMPALA, RDBMS]).order_by('-last_modified')

            for doc in docs:
                if doc.content_object:
                    notebook = import_saved_beeswax_query(doc.content_object)
                    data = notebook.get_data()

                    data['isSaved'] = False
                    data['snippets'][0]['lastExecuted'] = time.mktime(
                        doc.last_modified.timetuple()) * 1000

                    doc2 = _historify(data, self.user)
                    doc2.last_modified = doc.last_modified
                    doc2.save()

                    self.imported_docs.append(doc2)

                    # Tag for not re-importing
                    Document.objects.link(doc2,
                                          owner=doc2.owner,
                                          name=doc2.name,
                                          description=doc2.description,
                                          extra=doc.extra)

                    doc.add_tag(self.imported_tag)
                    doc.save()
        except ImportError, e:
            LOG.warn(
                'Cannot convert Saved Query documents: beeswax app is not installed'
            )