def queue_models(models, context):
""" Works an an appropriate ordering for the models.
This isn't essential, but makes the script look nicer because
more instances can be defined on their first try.
"""
# Max number of cycles allowed before we call it an infinite loop.
MAX_CYCLES = 5
model_queue = []
number_remaining_models = len(models)
allowed_cycles = MAX_CYCLES
while number_remaining_models > 0:
previous_number_remaining_models = number_remaining_models
model = models.pop(0)
# If the model is ready to be processed, add it to the list
if check_dependencies(model, model_queue):
model_class = ModelCode(model=model, context=context)
model_queue.append(model_class)
else:
# Otherwise put the model back at the end of the list
models.append(model)
# Check for infinite loops.
# This means there is a cyclic foreign key structure
# That cannot be resolved by re-ordering
number_remaining_models = len(models)
if number_remaining_models == previous_number_remaining_models:
allowed_cycles -= 1
if allowed_cycles <= 0:
# Add the remaining models, but do not remove them from the model list
missing_models = [ModelCode(model=m, context=context)
for m in models]
model_queue += missing_models
# Replace the models with the model class objects
# (sure, this is a little bit of hackery)
models[:] = missing_models
break
else:
allowed_cycles = MAX_CYCLES
return model_queue
def queue_models(models, context):
""" Works an an appropriate ordering for the models.
This isn't essential, but makes the script look nicer because
more instances can be defined on their first try.
"""
# Max number of cycles allowed before we call it an infinite loop.
MAX_CYCLES = 5
model_queue = []
number_remaining_models = len(models)
allowed_cycles = MAX_CYCLES
while number_remaining_models > 0:
previous_number_remaining_models = number_remaining_models
model = models.pop(0)
# If the model is ready to be processed, add it to the list
if check_dependencies(model, model_queue):
model_class = ModelCode(model=model, context=context)
model_queue.append(model_class)
# Otherwise put the model back at the end of the list
else:
models.append(model)
# Check for infinite loops.
# This means there is a cyclic foreign key structure
# That cannot be resolved by re-ordering
number_remaining_models = len(models)
if number_remaining_models == previous_number_remaining_models:
allowed_cycles -= 1
if allowed_cycles <= 0:
# Add the remaining models, but do not remove them from the model list
missing_models = [
ModelCode(model=m, context=context) for m in models
]
model_queue += missing_models
# Replace the models with the model class objects
# (sure, this is a little bit of hackery)
models[:] = missing_models
break
else:
allowed_cycles = MAX_CYCLES
return model_queue
def handle(self, *args, **options):
# Pick up the options
if 'database' in options:
self.database = options['database'] or DEFAULT_DB_ALIAS
else:
self.database = DEFAULT_DB_ALIAS
if self.database not in settings.DATABASES:
raise CommandError("No database settings known for '%s'" % self.database )
if 'user' in options and options['user']:
try:
self.user = User.objects.all().using(self.database).get(username=options['user'])
except:
raise CommandError("User '%s' not found" % options['user'] )
else:
self.user = None
now = datetime.now()
task = None
self.logfile = None
try:
# Initialize the task
if 'task' in options and options['task']:
try:
task = Task.objects.all().using(self.database).get(pk=options['task'])
except:
raise CommandError("Task identifier not found")
if task.started or task.finished or task.status != "Waiting" or task.name != 'load from folder':
raise CommandError("Invalid task identifier")
task.status = '0%'
task.started = now
else:
task = Task(name='load from folder', submitted=now, started=now, status='0%', user=self.user)
task.arguments = ' '.join(['"%s"' % i for i in args])
task.save(using=self.database)
# Choose the right self.delimiter and language
self.delimiter = get_format('DECIMAL_SEPARATOR', settings.LANGUAGE_CODE, True) == ',' and ';' or ','
translation.activate(settings.LANGUAGE_CODE)
# Execute
if os.path.isdir(settings.DATABASES[self.database]['FILEUPLOADFOLDER']):
# Open the logfile
self.logfile = open(os.path.join(settings.DATABASES[self.database]['FILEUPLOADFOLDER'], 'loadfromfolder.log'), "a")
print("%s Started upload from folder\n" % datetime.now(), file=self.logfile)
all_models = [ (ct.model_class(), ct.pk) for ct in ContentType.objects.all() if ct.model_class() ]
models = []
for ifile in os.listdir(settings.DATABASES[self.database]['FILEUPLOADFOLDER']):
if not ifile.endswith('.csv'):
continue
filename0 = ifile.split('.')[0]
model = None
contenttype_id = None
for m, ct in all_models:
if filename0.lower() in (m._meta.model_name.lower(), m._meta.verbose_name.lower(), m._meta.verbose_name_plural.lower()):
model = m
contenttype_id = ct
print("%s Matched a model to file: %s" % (datetime.now(),ifile), file=self.logfile)
break
if not model or model in EXCLUDE_FROM_BULK_OPERATIONS:
print("%s Ignoring data in file: %s" % (datetime.now(),ifile), file=self.logfile)
elif self.user and not self.user.has_perm('%s.%s' % (model._meta.app_label, get_permission_codename('add', model._meta))):
# Check permissions
print("%s You don't have permissions to add: %s" % (datetime.now(),ifile), file=self.logfile)
else:
deps = set([model])
GridReport.dependent_models(model, deps)
models.append( (ifile, model, contenttype_id, deps) )
# Sort the list of models, based on dependencies between models
cnt = len(models)
ok = False
while not ok:
ok = True
for i in range(cnt):
for j in range(i + 1, cnt):
if models[i][1] != models[j][1] and models[i][1] in models[j][3]:
# A subsequent model i depends on model i. The list ordering is
# thus not ok yet. We move this element to the end of the list.
models.append(models.pop(i))
ok = False
task.status = '10%'
task.save(using=self.database)
i=0
errors = 0
for ifile, model, contenttype_id, dependencies in models:
i += 1
print("%s Started processing data in file: %s" % (datetime.now(),ifile), file=self.logfile)
filetoparse=os.path.join(os.path.abspath(settings.DATABASES[self.database]['FILEUPLOADFOLDER']), ifile)
errors += self.parseCSVloadfromfolder(model, filetoparse)
print("%s Finished processing data in file: %s\n" % (datetime.now(),ifile), file=self.logfile)
task.status = str(int(10+i/cnt*80))+'%'
task.save(using=self.database)
# Task update
if errors:
task.status = 'Failed'
task.message = "Uploaded %s data files with %s errors" % (cnt, errors)
else:
task.status = 'Done'
task.message = "Uploaded %s data file" % cnt
task.finished = datetime.now()
except Exception as e:
print("%s Failed" % datetime.now(), file=self.logfile)
if task:
task.status = 'Failed'
task.message = '%s' % e
task.finished = datetime.now()
raise e
finally:
if task:
task.status = '100%'
task.save(using=self.database)
if self.logfile:
print('%s End of upload from folder\n' % datetime.now(), file=self.logfile)
self.logfile.close()
def handle(self, *args, **options):
# Pick up the options
if 'database' in options:
self.database = options['database'] or DEFAULT_DB_ALIAS
else:
self.database = DEFAULT_DB_ALIAS
if self.database not in settings.DATABASES:
raise CommandError("No database settings known for '%s'" % self.database )
if 'user' in options and options['user']:
try:
self.user = User.objects.all().using(self.database).get(username=options['user'])
except:
raise CommandError("User '%s' not found" % options['user'] )
else:
self.user = None
now = datetime.now()
task = None
try:
# Initialize the task
if 'task' in options and options['task']:
try:
task = Task.objects.all().using(self.database).get(pk=options['task'])
except:
raise CommandError("Task identifier not found")
if task.started or task.finished or task.status != "Waiting" or task.name != 'load from folder':
raise CommandError("Invalid task identifier")
task.status = '0%'
task.started = now
else:
task = Task(name='load from folder', submitted=now, started=now, status='0%', user=self.user)
task.arguments = ' '.join(['"%s"' % i for i in args])
task.save(using=self.database)
# Choose the right self.delimiter and language
self.delimiter = get_format('DECIMAL_SEPARATOR', settings.LANGUAGE_CODE, True) == ',' and ';' or ','
translation.activate(settings.LANGUAGE_CODE)
# Execute
filestoupload = list()
if os.path.isdir(settings.DATABASES[self.database]['FILEUPLOADFOLDER']):
thisfolder = settings.DATABASES[self.database]['FILEUPLOADFOLDER']
for fileindir in os.listdir(settings.DATABASES[self.database]['FILEUPLOADFOLDER']):
if fileindir.endswith('.csv'):
filestoupload.append(fileindir)
#filestoupload.append([file,strftime("%Y-%m-%d %H:%M:%S",localtime(os.stat(os.path.join(thisfolder, file)).st_mtime)),sizeof_fmt(os.stat(os.path.join(thisfolder, file)).st_size, 'B')])
all_models = [ (ct.model_class(), ct.pk) for ct in ContentType.objects.all() if ct.model_class() ]
models = []
for ifile in filestoupload:
filename0 = ifile.split('.')[0]
model = None
contenttype_id = None
for m, ct in all_models:
if filename0.lower() in (m._meta.model_name.lower(), m._meta.verbose_name.lower(), m._meta.verbose_name_plural.lower()):
model = m
contenttype_id = ct
break
if not model or model in EXCLUDE_FROM_BULK_OPERATIONS:
print("Ignoring data in file: %s" % ifile)
elif not self.user==None and not self.user.has_perm('%s.%s' % (model._meta.app_label, get_permission_codename('add', model._meta))):
# Check permissions
print("You don't permissions to add: %s" % ifile)
else:
deps = set([model])
GridReport.dependent_models(model, deps)
models.append( (ifile, model, contenttype_id, deps) )
# Sort the list of models, based on dependencies between models
cnt = len(models)
ok = False
while not ok:
ok = True
for i in range(cnt):
for j in range(i + 1, cnt):
if models[i][1] in models[j][3]:
# A subsequent model i depends on model i. The list ordering is
# thus not ok yet. We move this element to the end of the list.
models.append(models.pop(i))
ok = False
for ifile, model, contenttype_id, dependencies in models:
print("Processing data in file: %s" % ifile)
rownum = 0
has_pk_field = False
headers = []
uploadform = None
changed = 0
added = 0
numerrors = 0
#Will the permissions have to be checked table by table?
permname = get_permission_codename('add', model._meta)
if not self.user == None and not self.user.has_perm('%s.%s' % (model._meta.app_label, permname)):
print('Permission denied')
return
filetoparse=os.path.join(os.path.abspath(thisfolder), ifile)
self.parseCSVloadfromfolder(model, filetoparse)
# Task update
task.status = 'Done'
task.finished = datetime.now()
except Exception as e:
if task:
task.status = 'Failed'
task.message = '%s' % e
task.finished = datetime.now()
raise e
finally:
if task:
task.save(using=self.database)