예제 #1
0
def queue_models(models, context):
    """ Works an an appropriate ordering for the models.
        This isn't essential, but makes the script look nicer because 
        more instances can be defined on their first try.
    """

    # Max number of cycles allowed before we call it an infinite loop.

    MAX_CYCLES = 5

    model_queue = []
    number_remaining_models = len(models)
    allowed_cycles = MAX_CYCLES

    while number_remaining_models > 0:
        previous_number_remaining_models = number_remaining_models

        model = models.pop(0)

        # If the model is ready to be processed, add it to the list

        if check_dependencies(model, model_queue):
            model_class = ModelCode(model=model, context=context)
            model_queue.append(model_class)
        else:

        # Otherwise put the model back at the end of the list

            models.append(model)

        # Check for infinite loops.
        # This means there is a cyclic foreign key structure
        # That cannot be resolved by re-ordering

        number_remaining_models = len(models)
        if number_remaining_models == previous_number_remaining_models:
            allowed_cycles -= 1
            if allowed_cycles <= 0:

                # Add the remaining models, but do not remove them from the model list

                missing_models = [ModelCode(model=m, context=context)
                                  for m in models]
                model_queue += missing_models

                # Replace the models with the model class objects
                # (sure, this is a little bit of hackery)

                models[:] = missing_models
                break
        else:
            allowed_cycles = MAX_CYCLES

    return model_queue
예제 #2
0
def queue_models(models, context):
    """ Works an an appropriate ordering for the models.
    This isn't essential, but makes the script look nicer because
    more instances can be defined on their first try.
    """

    # Max number of cycles allowed before we call it an infinite loop.
    MAX_CYCLES = 5

    model_queue = []
    number_remaining_models = len(models)
    allowed_cycles = MAX_CYCLES

    while number_remaining_models > 0:
        previous_number_remaining_models = number_remaining_models

        model = models.pop(0)

        # If the model is ready to be processed, add it to the list
        if check_dependencies(model, model_queue):
            model_class = ModelCode(model=model, context=context)
            model_queue.append(model_class)

        # Otherwise put the model back at the end of the list
        else:
            models.append(model)

        # Check for infinite loops.
        # This means there is a cyclic foreign key structure
        # That cannot be resolved by re-ordering
        number_remaining_models = len(models)
        if number_remaining_models == previous_number_remaining_models:
            allowed_cycles -= 1
            if allowed_cycles <= 0:
                # Add the remaining models, but do not remove them from the model list
                missing_models = [
                    ModelCode(model=m, context=context) for m in models
                ]
                model_queue += missing_models
                # Replace the models with the model class objects
                # (sure, this is a little bit of hackery)
                models[:] = missing_models
                break
        else:
            allowed_cycles = MAX_CYCLES

    return model_queue
예제 #3
0
  def handle(self, *args, **options):
    # Pick up the options
    if 'database' in options:
      self.database = options['database'] or DEFAULT_DB_ALIAS
    else:
      self.database = DEFAULT_DB_ALIAS
    if self.database not in settings.DATABASES:
      raise CommandError("No database settings known for '%s'" % self.database )
    if 'user' in options and options['user']:
      try:
        self.user = User.objects.all().using(self.database).get(username=options['user'])
      except:
        raise CommandError("User '%s' not found" % options['user'] )
    else:
      self.user = None

    now = datetime.now()

    task = None
    self.logfile = None
    try:
      # Initialize the task
      if 'task' in options and options['task']:
        try:
          task = Task.objects.all().using(self.database).get(pk=options['task'])
        except:
          raise CommandError("Task identifier not found")
        if task.started or task.finished or task.status != "Waiting" or task.name != 'load from folder':
          raise CommandError("Invalid task identifier")
        task.status = '0%'
        task.started = now
      else:
        task = Task(name='load from folder', submitted=now, started=now, status='0%', user=self.user)
      task.arguments = ' '.join(['"%s"' % i for i in args])
      task.save(using=self.database)

      # Choose the right self.delimiter and language
      self.delimiter = get_format('DECIMAL_SEPARATOR', settings.LANGUAGE_CODE, True) == ',' and ';' or ','
      translation.activate(settings.LANGUAGE_CODE)

      # Execute
      if os.path.isdir(settings.DATABASES[self.database]['FILEUPLOADFOLDER']):

        # Open the logfile
        self.logfile = open(os.path.join(settings.DATABASES[self.database]['FILEUPLOADFOLDER'], 'loadfromfolder.log'), "a")
        print("%s Started upload from folder\n" % datetime.now(), file=self.logfile)

        all_models = [ (ct.model_class(), ct.pk) for ct in ContentType.objects.all() if ct.model_class() ]
        models = []
        for ifile in os.listdir(settings.DATABASES[self.database]['FILEUPLOADFOLDER']):
          if not ifile.endswith('.csv'):
            continue
          filename0 = ifile.split('.')[0]

          model = None
          contenttype_id = None
          for m, ct in all_models:
            if filename0.lower() in (m._meta.model_name.lower(), m._meta.verbose_name.lower(), m._meta.verbose_name_plural.lower()):
              model = m
              contenttype_id = ct
              print("%s Matched a model to file: %s" % (datetime.now(),ifile), file=self.logfile)
              break

          if not model or model in EXCLUDE_FROM_BULK_OPERATIONS:
            print("%s Ignoring data in file: %s" % (datetime.now(),ifile), file=self.logfile)
          elif self.user and not self.user.has_perm('%s.%s' % (model._meta.app_label, get_permission_codename('add', model._meta))):
            # Check permissions
            print("%s You don't have permissions to add: %s" % (datetime.now(),ifile), file=self.logfile)
          else:
            deps = set([model])
            GridReport.dependent_models(model, deps)

            models.append( (ifile, model, contenttype_id, deps) )

        # Sort the list of models, based on dependencies between models
        cnt = len(models)
        ok = False
        while not ok:
          ok = True
          for i in range(cnt):
            for j in range(i + 1, cnt):
              if models[i][1] != models[j][1] and models[i][1] in models[j][3]:
                # A subsequent model i depends on model i. The list ordering is
                # thus not ok yet. We move this element to the end of the list.
                models.append(models.pop(i))
                ok = False
        task.status = '10%'
        task.save(using=self.database)

        i=0
        errors = 0
        for ifile, model, contenttype_id, dependencies in models:
          i += 1
          print("%s Started processing data in file: %s" % (datetime.now(),ifile), file=self.logfile)
          filetoparse=os.path.join(os.path.abspath(settings.DATABASES[self.database]['FILEUPLOADFOLDER']), ifile)
          errors += self.parseCSVloadfromfolder(model, filetoparse)
          print("%s Finished processing data in file: %s\n" % (datetime.now(),ifile), file=self.logfile)
          task.status = str(int(10+i/cnt*80))+'%'
          task.save(using=self.database)

      # Task update
      if errors:
        task.status = 'Failed'
        task.message = "Uploaded %s data files with %s errors" % (cnt, errors)
      else:
        task.status = 'Done'
        task.message = "Uploaded %s data file" % cnt
      task.finished = datetime.now()

    except Exception as e:
      print("%s Failed" % datetime.now(), file=self.logfile)
      if task:
        task.status = 'Failed'
        task.message = '%s' % e
        task.finished = datetime.now()
      raise e

    finally:
      if task:
        task.status = '100%'
        task.save(using=self.database)
      if self.logfile:
        print('%s End of upload from folder\n' % datetime.now(), file=self.logfile)
        self.logfile.close()
  def handle(self, *args, **options):
    # Pick up the options
    if 'database' in options:
      self.database = options['database'] or DEFAULT_DB_ALIAS
    else:
      self.database = DEFAULT_DB_ALIAS
    if self.database not in settings.DATABASES:
      raise CommandError("No database settings known for '%s'" % self.database )
    if 'user' in options and options['user']:
      try:
        self.user = User.objects.all().using(self.database).get(username=options['user'])
      except:
        raise CommandError("User '%s' not found" % options['user'] )
    else:
      self.user = None

    now = datetime.now()

    task = None
    try:
      # Initialize the task
      if 'task' in options and options['task']:
        try:
          task = Task.objects.all().using(self.database).get(pk=options['task'])
        except:
          raise CommandError("Task identifier not found")
        if task.started or task.finished or task.status != "Waiting" or task.name != 'load from folder':
          raise CommandError("Invalid task identifier")
        task.status = '0%'
        task.started = now
      else:
        task = Task(name='load from folder', submitted=now, started=now, status='0%', user=self.user)
      task.arguments = ' '.join(['"%s"' % i for i in args])
      task.save(using=self.database)
      
      # Choose the right self.delimiter and language
      self.delimiter = get_format('DECIMAL_SEPARATOR', settings.LANGUAGE_CODE, True) == ',' and ';' or ','
      translation.activate(settings.LANGUAGE_CODE)
      
      # Execute        
      filestoupload = list()
      if os.path.isdir(settings.DATABASES[self.database]['FILEUPLOADFOLDER']):
        thisfolder = settings.DATABASES[self.database]['FILEUPLOADFOLDER']
        for fileindir in os.listdir(settings.DATABASES[self.database]['FILEUPLOADFOLDER']):
          if fileindir.endswith('.csv'):
            filestoupload.append(fileindir)
            #filestoupload.append([file,strftime("%Y-%m-%d %H:%M:%S",localtime(os.stat(os.path.join(thisfolder, file)).st_mtime)),sizeof_fmt(os.stat(os.path.join(thisfolder, file)).st_size, 'B')])
        
        all_models = [ (ct.model_class(), ct.pk) for ct in ContentType.objects.all() if ct.model_class() ]
        models = []
        for ifile in filestoupload:
          
          filename0 = ifile.split('.')[0]
          
          model = None
          contenttype_id = None
          for m, ct in all_models:
            if filename0.lower() in (m._meta.model_name.lower(), m._meta.verbose_name.lower(), m._meta.verbose_name_plural.lower()):
              model = m
              contenttype_id = ct
              break
            
          if not model or model in EXCLUDE_FROM_BULK_OPERATIONS:
            print("Ignoring data in file: %s" % ifile)
          elif not self.user==None and not self.user.has_perm('%s.%s' % (model._meta.app_label, get_permission_codename('add', model._meta))):
            # Check permissions
            print("You don't permissions to add: %s" % ifile)
          else:
            deps = set([model])
            GridReport.dependent_models(model, deps)
            models.append( (ifile, model, contenttype_id, deps) )

    # Sort the list of models, based on dependencies between models
        cnt = len(models)
        ok = False
        while not ok:
          ok = True
          for i in range(cnt):
            for j in range(i + 1, cnt):
              if models[i][1] in models[j][3]:
                # A subsequent model i depends on model i. The list ordering is
                # thus not ok yet. We move this element to the end of the list.
                models.append(models.pop(i))
                ok = False

        for ifile, model, contenttype_id, dependencies in models:
          
          print("Processing data in file: %s" % ifile)
          rownum = 0
          has_pk_field = False
          headers = []
          uploadform = None
          changed = 0
          added = 0
          numerrors = 0
          
          #Will the permissions have to be checked table by table?
          permname = get_permission_codename('add', model._meta)
          if not self.user == None and not self.user.has_perm('%s.%s' % (model._meta.app_label, permname)):
            print('Permission denied')
            return
          

          filetoparse=os.path.join(os.path.abspath(thisfolder), ifile)
          self.parseCSVloadfromfolder(model, filetoparse)
            
      # Task update
      task.status = 'Done'
      task.finished = datetime.now()

    except Exception as e:
      if task:
        task.status = 'Failed'
        task.message = '%s' % e
        task.finished = datetime.now()
      raise e

    finally:
      if task:
        task.save(using=self.database)