示例#1
0
  def _convert_saved_queries(self):
    # Convert SavedQuery documents
    try:
      from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS

      docs = self._get_unconverted_docs(SavedQuery).filter(extra__in=[HQL, IMPALA, RDBMS])
      for doc in docs:
        try:
          if doc.content_object:
            notebook = import_saved_beeswax_query(doc.content_object)
            data = notebook.get_data()

            doc2 = self._create_doc2(
              document=doc,
              doctype=data['type'],
              name=data['name'],
              description=data['description'],
              data=notebook.get_json()
            )

            self.imported_doc_count += 1
        except Exception, e:
          self.failed_doc_ids.append(doc.id)
          LOG.exception('Failed to import SavedQuery document id: %d' % doc.id)
    except ImportError:
      LOG.warn('Cannot convert Saved Query documents: beeswax app is not installed')
示例#2
0
    def _convert_saved_queries(self):
        # Convert SavedQuery documents
        try:
            from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS

            docs = self._get_unconverted_docs(SavedQuery).filter(
                extra__in=[HQL, IMPALA, RDBMS])
            for doc in docs:
                try:
                    if doc.content_object:
                        notebook = import_saved_beeswax_query(
                            doc.content_object)
                        data = notebook.get_data()

                        doc2 = self._create_doc2(
                            document=doc,
                            doctype=data['type'],
                            name=data['name'],
                            description=data['description'],
                            data=notebook.get_json())

                        self.imported_doc_count += 1
                except Exception, e:
                    self.failed_doc_ids.append(doc.id)
                    LOG.exception(
                        'Failed to import SavedQuery document id: %d' % doc.id)
        except ImportError:
            LOG.warn(
                'Cannot convert Saved Query documents: beeswax app is not installed'
            )
示例#3
0
文件: tests_doc2.py 项目: soitun/hue
  def test_document_create(self):
    sql = 'SELECT * FROM sample_07'

    design = hql_query(sql)

    # is_auto
    # is_trashed
    # is_redacted
    old_query = SavedQuery.objects.create(
        type=SavedQuery.TYPES_MAPPING['hql'],
        owner=self.user,
        data=design.dumps(),
        name='See examples',
        desc='Example of old format'
    )

    try:
      new_query = import_saved_beeswax_query(old_query)
      new_query_data = new_query.get_data()

      assert_equal('query-hive', new_query_data['type'])
      assert_equal('See examples', new_query_data['name'])
      assert_equal('Example of old format', new_query_data['description'])

      assert_equal('ready', new_query_data['snippets'][0]['status'])
      assert_equal('See examples', new_query_data['snippets'][0]['name'])
      assert_equal('SELECT * FROM sample_07', new_query_data['snippets'][0]['statement_raw'])

      assert_equal([], new_query_data['snippets'][0]['properties']['settings'])
      assert_equal([], new_query_data['snippets'][0]['properties']['files'])
      assert_equal([], new_query_data['snippets'][0]['properties']['functions'])
    finally:
      old_query.delete()
示例#4
0
  def test_document_create(self):
    sql = 'SELECT * FROM sample_07'

    design = hql_query(sql)

    # is_auto
    # is_trashed
    # is_redacted
    old_query = SavedQuery.objects.create(
        type=SavedQuery.TYPES_MAPPING['hql'],
        owner=self.user,
        data=design.dumps(),
        name='See examples',
        desc='Example of old format'
    )

    try:
      new_query = import_saved_beeswax_query(old_query)
      new_query_data = new_query.get_data()

      assert_equal('query-hive', new_query_data['type'])
      assert_equal('See examples', new_query_data['name'])
      assert_equal('Example of old format', new_query_data['description'])

      assert_equal('ready', new_query_data['snippets'][0]['status'])
      assert_equal('See examples', new_query_data['snippets'][0]['name'])
      assert_equal('SELECT * FROM sample_07', new_query_data['snippets'][0]['statement_raw'])

      assert_equal([], new_query_data['snippets'][0]['properties']['settings'])
      assert_equal([], new_query_data['snippets'][0]['properties']['files'])
      assert_equal([], new_query_data['snippets'][0]['properties']['functions'])
    finally:
      old_query.delete()
示例#5
0
文件: converters.py 项目: XOEEst/hue
  def convert(self):
    # Convert SavedQuery documents
    try:
      from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS

      docs = self._get_unconverted_docs(SavedQuery).filter(extra__in=[HQL, IMPALA, RDBMS])
      for doc in docs:
        if doc.content_object:
          notebook = import_saved_beeswax_query(doc.content_object)
          data = notebook.get_data()

          if doc.is_historic():
            data['isSaved'] = False

          doc2 = self._create_doc2(
              document=doc,
              doctype=data['type'],
              name=data['name'],
              description=data['description'],
              data=notebook.get_json()
          )

          if doc.is_historic():
            doc2.is_history = False

          self.imported_docs.append(doc2)
    except ImportError, e:
      LOG.warn('Cannot convert Saved Query documents: beeswax app is not installed')
示例#6
0
  def _convert_query_histories(self):
    # Convert SQL Query history documents
    try:
      from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS

      docs = self._get_unconverted_docs(SavedQuery, only_history=True).filter(extra__in=[HQL, IMPALA, RDBMS]).order_by(
        '-last_modified')

      for doc in docs:
        try:
          if doc.content_object:
            notebook = import_saved_beeswax_query(doc.content_object)
            data = notebook.get_data()

            data['isSaved'] = False
            data['snippets'][0]['lastExecuted'] = time.mktime(doc.last_modified.timetuple()) * 1000

            with transaction.atomic():
              doc2 = _historify(data, self.user)
              doc2.last_modified = doc.last_modified

              # save() updates the last_modified to current time. Resetting it using update()
              doc2.save()
              Document2.objects.filter(id=doc2.id).update(last_modified=doc.last_modified)

              self.imported_doc_count += 1

              doc.add_tag(self.imported_tag)
              doc.save()
        except Exception, e:
          self.failed_doc_ids.append(doc.id)
          LOG.exception('Failed to import history document id: %d' % doc.id)
    except ImportError, e:
      LOG.warn('Cannot convert history documents: beeswax app is not installed')
示例#7
0
    def convert(self):
        # Convert SavedQuery documents
        try:
            from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS

            docs = self._get_unconverted_docs(SavedQuery).filter(
                extra__in=[HQL, IMPALA, RDBMS])
            for doc in docs:
                if doc.content_object:
                    notebook = import_saved_beeswax_query(doc.content_object)
                    data = notebook.get_data()

                    if doc.is_historic():
                        data['isSaved'] = False

                    doc2 = self._create_doc2(document=doc,
                                             doctype=data['type'],
                                             name=data['name'],
                                             description=data['description'],
                                             data=notebook.get_json())

                    if doc.is_historic():
                        doc2.is_history = False

                    self.imported_docs.append(doc2)
        except ImportError, e:
            LOG.warn(
                'Cannot convert Saved Query documents: beeswax app is not installed'
            )
示例#8
0
  def install(self, django_user, interpreter=None):
    """
    Install queries. Raise InstallException on failure.
    """
    LOG.info('Installing sample query: %s' % (self.name,))

    try:
      # Don't overwrite
      query = SavedQuery.objects.get(owner=django_user, name=self.name, type=self.type)
    except SavedQuery.DoesNotExist:
      query = SavedQuery(owner=django_user, name=self.name, type=self.type, desc=self.desc)
      # The data field needs to be a string. The sample file writes it as json (without encoding into a string) for readability.
      query.data = json.dumps(self.data)
      query.save()
      LOG.info('Successfully installed sample design: %s' % (self.name,))

    if USE_NEW_EDITOR.get():
      examples_dir = _get_example_directory(django_user)

      document_type = self._document_type(self.type, interpreter)
      notebook = import_saved_beeswax_query(query, interpreter=interpreter)

      try:
        # Could move PK from a uuid in queries.json at some point to handle name changes without duplicating.
        # And move to a simpler json format at some point.
        doc2 = Document2.objects.get(owner=django_user, name=self.name, type=document_type, is_history=False)

        if doc2.parent_directory != examples_dir:  # Recover from Trash or if moved
          doc2.parent_directory = examples_dir

        data = json.loads(doc2.data)
        data['uuid'] = doc2.uuid
        doc2.data = json.dumps(data)  # Refresh content

        doc2.save()
      except Document2.DoesNotExist:
        # Create new example
        data = notebook.get_data()
        data['isSaved'] = True
        uuid = data.get('uuid')
        data = json.dumps(data)

        doc2 = Document2.objects.create(
          uuid=uuid,  # Must the same as in the notebook data
          owner=django_user,
          parent_directory=examples_dir,
          name=self.name,
          type=document_type,
          description=self.desc,
          data=data
        )

        # TODO: FK to connector object

      # Share with default group
      examples_dir.share(django_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
      LOG.info('Successfully installed sample query: %s' % doc2)
示例#9
0
  def install(self, django_user, interpreter=None):
    """
    Install queries. Raise InstallException on failure.
    """
    LOG.info('Installing sample query: %s' % (self.name,))

    try:
      # Don't overwrite
      query = SavedQuery.objects.get(owner=django_user, name=self.name, type=self.type)
    except SavedQuery.DoesNotExist:
      query = SavedQuery(owner=django_user, name=self.name, type=self.type, desc=self.desc)
      # The data field needs to be a string. The sample file writes it
      # as json (without encoding into a string) for readability.
      query.data = json.dumps(self.data)
      query.save()
      LOG.info('Successfully installed sample design: %s' % (self.name,))

    if USE_NEW_EDITOR.get():
      # Get or create sample user directories
      home_dir = Directory.objects.get_home_directory(django_user)
      examples_dir, created = Directory.objects.get_or_create(
        parent_directory=home_dir,
        owner=django_user,
        name=Document2.EXAMPLES_DIR
      )

      document_type = self._document_type(self.type, interpreter)
      try:
        # Don't overwrite
        doc2 = Document2.objects.get(owner=django_user, name=self.name, type=document_type, is_history=False)
        # If document exists but has been trashed, recover from Trash
        if doc2.parent_directory != examples_dir:
          doc2.parent_directory = examples_dir
          doc2.save()
      except Document2.DoesNotExist:
        # Create document from saved query
        notebook = import_saved_beeswax_query(query, interpreter=interpreter)
        data = notebook.get_data()
        data['isSaved'] = True
        uuid = data.get('uuid')
        data = json.dumps(data)

        doc2 = Document2.objects.create(
          uuid=uuid,
          owner=django_user,
          parent_directory=examples_dir,
          name=self.name,
          type=document_type,
          description=self.desc,
          data=data
        )

      # Share with default group
      examples_dir.share(django_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
      LOG.info('Successfully installed sample query: %s' % (self.name,))
示例#10
0
  def install(self, django_user):
    """
    Install queries. Raise InstallException on failure.
    """
    LOG.info('Installing sample query: %s' % (self.name,))

    try:
      # Don't overwrite
      query = SavedQuery.objects.get(owner=django_user, name=self.name, type=self.type)
    except SavedQuery.DoesNotExist:
      query = SavedQuery(owner=django_user, name=self.name, type=self.type, desc=self.desc)
      # The data field needs to be a string. The sample file writes it
      # as json (without encoding into a string) for readability.
      query.data = json.dumps(self.data)
      query.save()
      LOG.info('Successfully installed sample design: %s' % (self.name,))

    if USE_NEW_EDITOR.get():
      # Get or create sample user directories
      home_dir = Directory.objects.get_home_directory(django_user)
      examples_dir, created = Directory.objects.get_or_create(
        parent_directory=home_dir,
        owner=django_user,
        name=Document2.EXAMPLES_DIR
      )

      try:
        # Don't overwrite
        doc2 = Document2.objects.get(owner=django_user, name=self.name, type=self._document_type(self.type))
        # If document exists but has been trashed, recover from Trash
        if doc2.parent_directory != examples_dir:
          doc2.parent_directory = examples_dir
          doc2.save()
      except Document2.DoesNotExist:
        # Create document from saved query
        notebook = import_saved_beeswax_query(query)
        data = notebook.get_data()
        data['isSaved'] = True
        uuid = data.get('uuid')
        data = json.dumps(data)

        doc2 = Document2.objects.create(
          uuid=uuid,
          owner=django_user,
          parent_directory=examples_dir,
          name=self.name,
          type=self._document_type(self.type),
          description=self.desc,
          data=data
        )

      # Share with default group
      examples_dir.share(django_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
      LOG.info('Successfully installed sample query: %s' % (self.name,))
示例#11
0
    def _convert_query_histories(self):
        # Convert SQL Query history documents
        try:
            from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS

            docs = self._get_unconverted_docs(
                SavedQuery, only_history=True).filter(
                    extra__in=[HQL, IMPALA, RDBMS]).order_by('-last_modified')

            for doc in docs:
                try:
                    if doc.content_object:
                        notebook = import_saved_beeswax_query(
                            doc.content_object)
                        data = notebook.get_data()

                        data['isSaved'] = False
                        data['snippets'][0]['lastExecuted'] = time.mktime(
                            doc.last_modified.timetuple()) * 1000

                        with transaction.atomic():
                            doc2 = _historify(data, self.user)
                            doc2.last_modified = doc.last_modified

                            # save() updates the last_modified to current time. Resetting it using update()
                            doc2.save()
                            Document2.objects.filter(id=doc2.id).update(
                                last_modified=doc.last_modified)

                            self.imported_doc_count += 1

                            doc.add_tag(self.imported_tag)
                            doc.save()
                except Exception, e:
                    self.failed_doc_ids.append(doc.id)
                    LOG.exception('Failed to import history document id: %d' %
                                  doc.id)
        except ImportError, e:
            LOG.warn(
                'Cannot convert history documents: beeswax app is not installed'
            )
示例#12
0
    def convert(self):
        # Convert SavedQuery documents
        try:
            from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS

            docs = self._get_unconverted_docs(SavedQuery).filter(
                extra__in=[HQL, IMPALA, RDBMS])
            for doc in docs:
                if doc.content_object:
                    notebook = import_saved_beeswax_query(doc.content_object)
                    data = notebook.get_data()

                    if doc.is_historic():
                        data['isSaved'] = False

                    doc2 = self._create_doc2(document=doc,
                                             doctype=data['type'],
                                             name=data['name'],
                                             description=data['description'],
                                             data=notebook.get_json())

                    if doc.is_historic():
                        doc2.is_history = False

                    self.imported_docs.append(doc2)
        except ImportError:
            LOG.warn(
                'Cannot convert Saved Query documents: beeswax app is not installed'
            )

        # Convert SQL Query history documents
        try:
            from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS

            docs = self._get_unconverted_docs(
                SavedQuery, with_history=True).filter(
                    extra__in=[HQL, IMPALA, RDBMS]).order_by('-last_modified')

            for doc in docs:
                if doc.content_object:
                    notebook = import_saved_beeswax_query(doc.content_object)
                    data = notebook.get_data()

                    data['isSaved'] = False
                    data['snippets'][0]['lastExecuted'] = time.mktime(
                        doc.last_modified.timetuple()) * 1000

                    doc2 = _historify(data, self.user)
                    doc2.last_modified = doc.last_modified

                    # save() updates the last_modified to current time. Resetting it using update()
                    doc2.save()
                    Document2.objects.filter(id=doc2.id).update(
                        last_modified=doc.last_modified)

                    self.imported_docs.append(doc2)

                    # Tag for not re-importing
                    Document.objects.link(doc2,
                                          owner=doc2.owner,
                                          name=doc2.name,
                                          description=doc2.description,
                                          extra=doc.extra)

                    doc.add_tag(self.imported_tag)
                    doc.save()
        except ImportError, e:
            LOG.warn(
                'Cannot convert Saved Query documents: beeswax app is not installed'
            )
  def convertfailed(self):
    # Convert SavedQuery documents
    try:
      from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
  
      docs = self._get_unconverted_docs(SavedQuery).filter(extra__in=[HQL, IMPALA, RDBMS])
      for doc in docs:
        if doc.content_object:
          id_temp = doc.to_dict()
          id = id_temp['id']
          notebook = import_saved_beeswax_query(doc.content_object)
          data = notebook.get_data()
          name = data['name']
          query = data['snippets'][0]['statement_raw']
          if re.match(self.startqueryname, name) and not self.startuser:
            self.processdocs = True
          if self.processdocs:
            matchdocs = findMatchingQuery(user=self.user, id=id, name=name, query=query, include_history=False)
            if not matchdocs or self.allowdupes:
              try:
                if doc.is_historic():
                  data['isSaved'] = False

                doc2 = self._create_doc2(
                    document=doc,
                    doctype=data['type'],
                    name=data['name'],
                    description=data['description'],
                    data=notebook.get_json()
                )

                if doc.is_historic():
                  doc2.is_history = False

                self.imported_docs.append(doc2)
              
              except:
                pass

    except ImportError:
      LOG.info('Cannot convert Saved Query documents: beeswax app is not installed')
      pass

    # Convert SQL Query history documents
    try:
      from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS

      docs = self._get_unconverted_docs(SavedQuery, with_history=True).filter(extra__in=[HQL, IMPALA, RDBMS]).order_by('-last_modified')

      for doc in docs:
  	if not doc.content_object:
          LOG.error("Content object is missing")
        elif doc.content_object:
          id_temp = doc.to_dict()
          id = id_temp['id']
          notebook = import_saved_beeswax_query(doc.content_object)
          data = notebook.get_data()
          name = data['name']
          query = data['snippets'][0]['statement_raw']
          if re.match(self.startqueryname, name) and not self.startuser:
            self.processdocs = True
          if self.processdocs:
            try:
              data['isSaved'] = False
              data['snippets'][0]['lastExecuted'] = time.mktime(doc.last_modified.timetuple()) * 1000

              doc2 = self._historify(data, self.user)
              doc2.last_modified = doc.last_modified

              # save() updates the last_modified to current time. Resetting it using update()
              doc2.save()
              Document2.objects.filter(id=doc2.id).update(last_modified=doc.last_modified)
 
              self.imported_docs.append(doc2)
  
                # Tag for not re-importing
              Document.objects.link(
                doc2,
                owner=doc2.owner,
                name=doc2.name,
                description=doc2.description,
                extra=doc.extra
              )
  
              try:
                doc.add_tag(self.imported_tag)
              except IntegrityError, e:
                LOG.exception("Failed to add imported_tag to doc %s with error %s" % (doc2.name, e))
                pass

              doc.save()

            except:
示例#14
0
  def convert(self):
    # Convert SavedQuery documents
    try:
      from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS

      docs = self._get_unconverted_docs(SavedQuery).filter(extra__in=[HQL, IMPALA, RDBMS])
      for doc in docs:
        if doc.content_object:
          notebook = import_saved_beeswax_query(doc.content_object)
          data = notebook.get_data()

          if doc.is_historic():
            data['isSaved'] = False

          doc2 = self._create_doc2(
              document=doc,
              doctype=data['type'],
              name=data['name'],
              description=data['description'],
              data=notebook.get_json()
          )

          if doc.is_historic():
            doc2.is_history = False

          self.imported_docs.append(doc2)
    except ImportError:
      LOG.warn('Cannot convert Saved Query documents: beeswax app is not installed')

    # Convert SQL Query history documents
    try:
      from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS

      docs = self._get_unconverted_docs(SavedQuery, with_history=True).filter(extra__in=[HQL, IMPALA, RDBMS]).order_by('-last_modified')

      for doc in docs:
        if doc.content_object:
          notebook = import_saved_beeswax_query(doc.content_object)
          data = notebook.get_data()

          data['isSaved'] = False
          data['snippets'][0]['lastExecuted'] = time.mktime(doc.last_modified.timetuple()) * 1000

          doc2 = _historify(data, self.user)
          doc2.last_modified = doc.last_modified
          doc2.save()

          self.imported_docs.append(doc2)

          # Tag for not re-importing
          Document.objects.link(
            doc2,
            owner=doc2.owner,
            name=doc2.name,
            description=doc2.description,
            extra=doc.extra
          )

          doc.add_tag(self.imported_tag)
          doc.save()
    except ImportError, e:
      LOG.warn('Cannot convert Saved Query documents: beeswax app is not installed')
示例#15
0
class DocumentConverter(object):
    """
  Given a user, converts any existing Document objects to Document2 objects
  """
    def __init__(self, user):
        self.user = user
        # If user does not have a home directory, we need to create one and import any orphan documents to it
        self.home_dir = Document2.objects.create_user_directories(self.user)
        self.imported_tag = DocumentTag.objects.get_imported2_tag(
            user=self.user)
        self.imported_docs = []

    def convert(self):
        # Convert SavedQuery documents
        try:
            from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS

            docs = self._get_unconverted_docs(SavedQuery).filter(
                extra__in=[HQL, IMPALA, RDBMS])
            for doc in docs:
                if doc.content_object:
                    notebook = import_saved_beeswax_query(doc.content_object)
                    data = notebook.get_data()

                    if doc.is_historic():
                        data['isSaved'] = False

                    doc2 = self._create_doc2(document=doc,
                                             doctype=data['type'],
                                             name=data['name'],
                                             description=data['description'],
                                             data=notebook.get_json())

                    if doc.is_historic():
                        doc2.is_history = False

                    self.imported_docs.append(doc2)
        except ImportError, e:
            LOG.warn(
                'Cannot convert Saved Query documents: beeswax app is not installed'
            )

        # Convert SQL Query history documents
        try:
            from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS

            docs = self._get_unconverted_docs(
                SavedQuery, with_history=True).filter(
                    extra__in=[HQL, IMPALA, RDBMS]).order_by('-last_modified')

            for doc in docs:
                if doc.content_object:
                    notebook = import_saved_beeswax_query(doc.content_object)
                    data = notebook.get_data()

                    data['isSaved'] = False
                    data['snippets'][0]['lastExecuted'] = time.mktime(
                        doc.last_modified.timetuple()) * 1000

                    doc2 = _historify(data, self.user)
                    doc2.last_modified = doc.last_modified
                    doc2.save()

                    self.imported_docs.append(doc2)

                    # Tag for not re-importing
                    Document.objects.link(doc2,
                                          owner=doc2.owner,
                                          name=doc2.name,
                                          description=doc2.description,
                                          extra=doc.extra)

                    doc.add_tag(self.imported_tag)
                    doc.save()
        except ImportError, e:
            LOG.warn(
                'Cannot convert Saved Query documents: beeswax app is not installed'
            )