def handle(self, dry_run=False, **options): verbosity = int(options['verbosity']) verbose = verbosity > 1 # start transaction transaction.enter_transaction_management() transaction.managed(True) count = 0 for document in models.Document.objects.filter(text_extracted=False): if verbose: print "PROCESS", repr(document) process_document_text(document) count += 1 if verbose: print "\nIN SUMMARY".ljust(70, '=') print count, "documents records processed" print "\n" if dry_run: transaction.rollback() else: transaction.commit() transaction.leave_transaction_management()
def _force_leave(db_name): try: transaction.set_clean(using=db_name) transaction.leave_transaction_management(using=db_name) except: _logger.error('force leave error: %s', db_name) _logger.except_error()
def process_response(self, request, response): """Commits and leaves transaction management.""" if transaction.is_managed(using=self.get_tenant(request)): if transaction.is_dirty(using=self.get_tenant(request)): transaction.commit(using=self.get_tenant(request)) transaction.leave_transaction_management(using=self.get_tenant(request)) return response
def commit(self): if self.done: return self.done = True transaction.commit(using=self.db) transaction.leave_transaction_management(using=self.db) print "Commiting"
def update(self, **kwargs): """ Updates all elements in the current QuerySet, setting all the given fields to the appropriate values. """ assert self.query.can_filter(), \ "Cannot update a query once a slice has been taken." query = self.query.clone(sql.UpdateQuery) query.add_update_values(kwargs) if not transaction.is_managed(): transaction.enter_transaction_management() forced_managed = True else: forced_managed = False try: rows = query.execute_sql(None) if forced_managed: transaction.commit() else: transaction.commit_unless_managed() finally: if forced_managed: transaction.leave_transaction_management() self._result_cache = None return rows
def _fixture_setup(self): transaction.commit_unless_managed() transaction.enter_transaction_management() transaction.managed(True) super(SeleniumTestCase, self)._fixture_setup() transaction.commit() transaction.leave_transaction_management()
def handle(self, **options): '''The main entry point for the Django management command.''' import_start = datetime.datetime.now() # Start transaction management. transaction.commit_unless_managed() transaction.enter_transaction_management() transaction.managed(True) try: self._promote_devel() except: self._rollback_db() raise # Finalize the transaction and close the db connection. transaction.commit() transaction.leave_transaction_management() connection.close() import_end = datetime.datetime.now() # Print a short summary of what we did. td = import_end - import_start print '\nProcessing complete in %s days, %s.%s seconds.' % ( td.days, td.seconds, td.microseconds) print ' TraitData objects promoted: %s' % ( PublicTraitData.objects.all().count(),)
def test_savepoint_rollback(self): """Tests rollbacks of savepoints""" from django.db import transaction from testapp.models import Genre, Publisher from johnny import cache if not connection.features.uses_savepoints: return self.failUnless(transaction.is_managed() == False) self.failUnless(transaction.is_dirty() == False) connection.queries = [] cache.local.clear() transaction.enter_transaction_management() transaction.managed() g = Genre.objects.get(pk=1) start_title = g.title g.title = "Adventures in Savepoint World" g.save() g = Genre.objects.get(pk=1) self.failUnless(g.title == "Adventures in Savepoint World") sid = transaction.savepoint() g.title = "In the Void" g.save() g = Genre.objects.get(pk=1) self.failUnless(g.title == "In the Void") transaction.savepoint_rollback(sid) g = Genre.objects.get(pk=1) self.failUnless(g.title == "Adventures in Savepoint World") transaction.rollback() g = Genre.objects.get(pk=1) self.failUnless(g.title == start_title) transaction.managed(False) transaction.leave_transaction_management()
def grab_db_lock(lock_name, wait): """ Grab a lock using a new, temporary connection. Yields a "success" boolean indicating whether the lock was successfully acquired or not. This context manager ensures that multidb does not override the connection when accessing the Lock model and its transaction. The code that runs within this context should NOT be affected - it must run as usual, with normal multidb functionality intact. """ with connections.get() as using: try: with connection_state.force(None): transaction.enter_transaction_management(using=using) transaction.managed(True, using=using) lock = Lock.grab(lock_name, wait=wait, using=using) success = bool(lock) try: yield success finally: if success: with connection_state.force(None): lock.release(using=using) transaction.commit(using=using) finally: with connection_state.force(None): transaction.leave_transaction_management(using=using)
def test_transaction_management(self): transaction.enter_transaction_management() self.assertFalse(connection.autocommit) self.assertEqual(connection.isolation_level, self._serializable) transaction.leave_transaction_management() self.assertTrue(connection.autocommit)
def tearDown(self): from django.db import transaction if transaction.is_managed(): if transaction.is_dirty(): transaction.rollback() transaction.managed(False) transaction.leave_transaction_management()
def atomic(using=None): """Perform database operations atomically within a transaction. The caller can use this to ensure SQL statements are executed within a transaction and then cleaned up nicely if there's an error. This provides compatibility with all supported versions of Django. Args: using (str, optional): The database connection name to use. Defaults to the default database connection. """ if hasattr(transaction, 'atomic'): # Django >= 1.5 with transaction.atomic(using=using): yield else: # Django < 1.5 assert hasattr(transaction, 'enter_transaction_management') try: # Begin Transaction transaction.enter_transaction_management(using=using) transaction.managed(True, using=using) yield transaction.commit(using=using) transaction.leave_transaction_management(using=using) except Exception: transaction.rollback(using=using) raise
def process_response(self, request, response): """Commits and leaves transaction management.""" if transaction.is_managed(): if transaction.is_dirty(): transaction.commit() transaction.leave_transaction_management() return response
def __call__(self, request, *args, **kwargs): """ By implementing the call method, we allow the class itself to act as the view function, taking in the request object and returning the response. This method is responsible for dealing with transaction management, dispatching to the proper instance methods, and returning a valid HttpResponse. """ if self.use_transactions: transaction.commit_unless_managed() transaction.enter_transaction_management() transaction.managed(True) context = self.view(request, *args, **kwargs) if isinstance(context, HttpResponseRedirect): return context context.update(self.extra_context) if self.next_field in request.REQUEST: next = request.REQUEST[self.next_field] if ':/' not in next: return HttpResponseRedirect(next) response = self.create_response(request, context) if self.use_transactions: transaction.commit() transaction.leave_transaction_management() return response
def execute_transaction(sql, output=False, database='default'): "A transaction wrapper for executing a list of SQL statements" my_connection = connection using_args = {} if is_multi_db(): if not database: database = DEFAULT_DB_ALIAS my_connection = connections[database] using_args['using'] = database try: # Begin Transaction transaction.enter_transaction_management(**using_args) transaction.managed(True, **using_args) cursor = my_connection.cursor() # Perform the SQL if output: write_sql(sql, database) execute_sql(cursor, sql) transaction.commit(**using_args) transaction.leave_transaction_management(**using_args) except Exception: transaction.rollback(**using_args) raise
def call_ranking(self, ranking, name, args): if ranking not in self.ranking_map: logging.warning('Attempted to call ranking, but not running: %s.%s', ranking.id, name) return logging.debug('Calling ranking: %s.%s', ranking.id, name) perf.begin('ranking') transaction.enter_transaction_management(True) transaction.managed(True) try: getattr(self.ranking_map[ranking], name)(*args) except: transaction.rollback() logging.exception('Ranking failed: %s', ranking.id) self.stop_ranking(ranking) try: ranking.header = 'Internal error' ranking.footer = '' ranking.save(force_update=True) RankingEntry.objects.filter(ranking=ranking).delete() except: transaction.rollback() logging.exception('Ranking cleanup failed: %s', ranking.id) transaction.commit() transaction.managed(False) transaction.leave_transaction_management() perf.end('ranking')
def serializerTest(format, self): # Clear the database first management.call_command('flush', verbosity=0, interactive=False) # Create all the objects defined in the test data objects = [] transaction.enter_transaction_management() transaction.managed(True) for (func, pk, klass, datum) in test_data: objects.append(func[0](pk, klass, datum)) transaction.commit() transaction.leave_transaction_management() # Add the generic tagged objects to the object list objects.extend(Tag.objects.all()) # Serialize the test database serialized_data = serializers.serialize(format, objects, indent=2) # Flush the database and recreate from the serialized data management.call_command('flush', verbosity=0, interactive=False) transaction.enter_transaction_management() transaction.managed(True) for obj in serializers.deserialize(format, serialized_data): obj.save() transaction.commit() transaction.leave_transaction_management() # Assert that the deserialized data is the same # as the original source for (func, pk, klass, datum) in test_data: func[1](self, pk, klass, datum)
def export_sales_order(self): transaction.enter_transaction_management(using=self.database) transaction.managed(True, using=self.database) try: starttime = time() if self.verbosity > 0: print("Exporting requested date of sales orders...") self.cursor.execute('''select substring(name from '^.*? '), max(plandate) from demand left outer join out_demand on demand.name = out_demand.demand and demand.subcategory = 'OpenERP' group by substring(name from '^.*? ') ''') cnt = 0 for i, j in self.cursor.fetchall(): result = self.sock.execute(self.openerp_db, self.uid, self.openerp_password, 'sale.order', 'write', [int(i)], {'requested_date': j and j.strftime('%Y-%m-%d') or 0,}) cnt += 1 if self.verbosity > 0: print("Updated %d sales orders in %.2f seconds" % (cnt, (time() - starttime))) except Exception as e: print("Error updating sales orders: %s" % e) finally: transaction.rollback(using=self.database) transaction.leave_transaction_management(using=self.database)
def call_test_suite_result(self, test_suite_result, name, args): if test_suite_result not in self.test_suite_result_map: logging.warning('Attempted to call test suite result, but not running: %s.%s', test_suite_result.id, name) return logging.debug('Calling test suite result: %s.%s', test_suite_result.id, name) transaction.enter_transaction_management(True) transaction.managed(True) try: getattr(self.test_suite_result_map[test_suite_result], name)(*args) except: transaction.rollback() logging.exception('Test suite result failed: %s', test_suite_result.id) self.finished_test_suite_result(test_suite_result) try: test_suite_result.status = 'INT' test_suite_result.report = 'Internal error' test_suite_result.save(force_update=True) except: transaction.rollback() logging.exception('Test suite result cleanup failed: %s', test_suite_result.id) transaction.commit() transaction.managed(False) transaction.leave_transaction_management()
def restore(self): self.state = BACKUP_RESTORE_STATE_IN_PROGRESS self.save() transaction.commit_unless_managed() transaction.enter_transaction_management() transaction.managed(True) try: self.tar = tarfile.open(self.backup_file.path, 'r:gz') # extract data.xml for parsing self.stream = self.tar.extractfile('backup/data.xml') self.restore_objects() self.restore_files() transaction.commit() transaction.leave_transaction_management() self.state = BACKUP_RESTORE_STATE_DONE except Exception as e: transaction.rollback() transaction.leave_transaction_management() self.state = BACKUP_RESTORE_STATE_ERROR self.error_message = e.__unicode__() mail_subject = _('Restore failed') mail_message = _('Restore for %(user)s failed with message : %(message)s') % {'user': self.user, 'message': e} mail_admins(mail_subject, mail_message, fail_silently=(not settings.DEBUG)) self.save() # close and delete archive self.tar.close() os.remove(self.backup_file.path) self.backup_file = None
def _commit_on_success_unless_managed(*args, **kw): try: if transaction.is_managed(): forced_managed = False else: transaction.enter_transaction_management() forced_managed = True try: res = func(*args, **kw) except: # All exceptions must be handled here (even string ones). if transaction.is_dirty(): if forced_managed: transaction.rollback() else: transaction.rollback_unless_managed() raise else: if transaction.is_dirty(): if forced_managed: transaction.commit() else: transaction.commit_unless_managed() return res finally: if forced_managed: transaction.leave_transaction_management()
def wrapped_func(*args, **kwargs): enter_transaction_management(using=using) managed(True, using=using) try: res = func(*args, **kwargs) except: if is_dirty(using=using): rollback(using=using) raise else: if is_dirty(using=using): if not isinstance(res, HttpResponse) or res.status_code < 200 or res.status_code >= 400: rollback(using=using) else: try: commit(using=using) except: rollback(using=using) raise finally: leave_transaction_management(using=using) return res
def test_transaction_rollback(self): """Tests johnny's handling of transaction rollbacks. Similar to the commit, this sets up a write to a db in a transaction, reads from it (to force a cache write of sometime), then rolls back.""" from Queue import Queue as queue from django.db import transaction from testapp.models import Genre, Publisher from johnny import cache if settings.DATABASE_ENGINE == 'sqlite3': print "\n Skipping test requiring multiple threads." return self.failUnless(transaction.is_managed() == False) self.failUnless(transaction.is_dirty() == False) connection.queries = [] cache.local.clear() q = queue() other = lambda x: self._run_threaded(x, q) # load some data start = Genre.objects.get(id=1) other('Genre.objects.get(id=1)') hit, ostart = q.get() # these should be the same and should have hit cache self.failUnless(hit) self.failUnless(ostart == start) # enter manual transaction management transaction.enter_transaction_management() transaction.managed() start.title = 'Jackie Chan Novels' # local invalidation, this key should hit the localstore! nowlen = len(cache.local) start.save() self.failUnless(nowlen != len(cache.local)) # perform a read OUTSIDE this transaction... it should still see the # old gen key, and should still find the "old" data other('Genre.objects.get(id=1)') hit, ostart = q.get() self.failUnless(hit) self.failUnless(ostart.title != start.title) # perform a READ inside the transaction; this should hit the localstore # but not the outside! nowlen = len(cache.local) start2 = Genre.objects.get(id=1) self.failUnless(start2.title == start.title) self.failUnless(len(cache.local) > nowlen) transaction.rollback() # we rollback, and flush all johnny keys related to this transaction # subsequent gets should STILL hit the cache in the other thread # and indeed, in this thread. self.failUnless(transaction.is_dirty() == False) other('Genre.objects.get(id=1)') hit, ostart = q.get() self.failUnless(hit) start = Genre.objects.get(id=1) self.failUnless(ostart.title == start.title) transaction.managed(False) transaction.leave_transaction_management()
def handle(self, *args, **options): print("Download zip-archive...") f = urlopen(IPGEOBASE_SOURCE_URL) buffer = BytesIO(f.read()) f.close() print("Unpacking...") zip_file = ZipFile(buffer) cities_file_read = _read_file(zip_file, 'cities.txt') cidr_optim_file_read = _read_file(zip_file, 'cidr_optim.txt') zip_file.close() buffer.close() print("Start updating...") list_cities = cities_file_read.decode(IPGEOBASE_CODING).split('\n') list_cidr_optim = \ cidr_optim_file_read.decode(IPGEOBASE_CODING).split('\n') lines = \ _get_cidr_optim_with_cities_lines(list_cidr_optim, list_cities) cursor = connection.cursor() transaction.enter_transaction_management() try: transaction.managed(True) print("Delete old rows in table ipgeobase...") cursor.execute(DELETE_SQL) print ("Write new data...") cursor.executemany(INSERT_SQL, [l for l in lines if l]) transaction.commit() except Exception as e: message = "The data not updated:", e if send_message: mail_admins(subject=ERROR_SUBJECT, message=message) raise CommandError(message) finally: transaction.rollback() transaction.leave_transaction_management() return "Table ipgeobase is update.\n"
def test_transaction_management(self): transaction.enter_transaction_management() transaction.managed(True) self.assertEqual(connection.isolation_level, self._read_committed) transaction.leave_transaction_management() self.assertEqual(connection.isolation_level, self._autocommit)
def handle(self, *args, **options): from django.db import transaction path, slug, name = args # Start transaction management. transaction.commit_unless_managed() transaction.enter_transaction_management() transaction.managed(True) book = Book.objects.get(slug=slug) root, ext = os.path.splitext(path) ext = ext.lower() if ext: ext = ext[1:] if ext == "zip": ext = "daisy" source_sha1 = BookMedia.read_source_sha1(path, ext) print "Source file SHA1:", source_sha1 try: assert source_sha1 bm = book.media.get(type=ext, source_sha1=source_sha1) print "Replacing media: %s (%s)" % (bm.name.encode("utf-8"), ext) except (AssertionError, BookMedia.DoesNotExist): bm = BookMedia(book=book, type=ext) print "Creating new media" bm.name = name bm.file.save(None, ExistingFile(path)) bm.save() transaction.commit() transaction.leave_transaction_management()
def handle(self, *args, **options): self.batch_size = options.get('batch_size', 50) self.preserve = options.get("preserve", False) self.index = options.get("index", False) self.newline = options.get("newline", False) if not self.index: old_realtime_indexing = getattr(settings, "REALTIME_INDEXING", None) #this is not recommended by the django manual, but in case of management command it seems to work settings.REALTIME_INDEXING = False transaction.enter_transaction_management() transaction.managed(True) for records_url in args: print("Processing %s" % records_url) errors = self.process_url(records_url, options) print("Processing %s Done" % records_url) if errors: print("%d error(s) when processing %s, check your log file." % (len(errors), records_url)) transaction.leave_transaction_management() if not self.index and old_realtime_indexing: settings.REALTIME_INDEXING = old_realtime_indexing
def handle_noargs(self, **options): transaction.enter_transaction_management() transaction.managed() for thes in Thesaurus.objects.all(): context = graph.get_context(URIRef(thes.uri)) with Term.objects.disable_mptt_updates(): # @UndefinedVariable for i,(s,_,o) in enumerate(graph.triples((None, URIRef("http://www.w3.org/2004/02/skos/core#narrower"), None), context=context)): print("%d - Thesaurus %s term pref label %s parent %s" % (i+1,thes.label, repr(o), repr(s))) parent_term = Term.objects.get(uri=unicode(s)) # @UndefinedVariable term = Term.objects.get(uri=unicode(o)) # @UndefinedVariable term.tree_id = thes.id term.parent = parent_term term.save() Term.objects.filter(parent=None, thesaurus=thes).update(tree_id=thes.id) # @UndefinedVariable print("Rebuilding tree %d" % thes.id) Term.objects.rebuild() # @UndefinedVariable transaction.commit() reset_queries() transaction.leave_transaction_management()
def export_sales_order(self, cursor): transaction.enter_transaction_management(using=self.database) try: starttime = time() if self.verbosity > 0: print("Exporting expected delivery date of sales orders...") cursor.execute('''select demand.source, max(plandate) from demand left outer join out_demand on demand.name = out_demand.demand where demand.subcategory = 'openbravo' and status = 'open' group by source ''') count = 0 body = [ '<?xml version="1.0" encoding="UTF-8"?>', '<ob:Openbravo xmlns:ob="http://www.openbravo.com">' ] for i in cursor.fetchall(): body.append('<OrderLine id="%s"><description>Planned delivery date %s</description></OrderLine>' % i) count += 1 if self.verbosity > 0 and count % 500 == 1: print('.', end="") if self.verbosity > 0: print ('') body.append('</ob:Openbravo>') self.post_data('/openbravo/ws/dal/OrderLine', '\n'.join(body)) if self.verbosity > 0: print("Updated %d sales orders in %.2f seconds" % (count, (time() - starttime))) except Exception as e: raise CommandError("Error updating sales orders: %s" % e) finally: transaction.rollback(using=self.database) transaction.leave_transaction_management(using=self.database)
def save(self): current_site = Site.objects.get_current() transaction.enter_transaction_management() transaction.managed(True) try: try: self.user.set_password(self.cleaned_data['new_password1']) self.user.save() message = loader.render_to_string('reset_password_email.txt', { 'user': self.user, 'raw_password': self.cleaned_data['new_password1'], 'full_server_url': 'http://%s/' % current_site.domain }) msg = EmailMessage( subject='New password for CHIRP Volunteer Tracker', body=message, to=[self.user.email] ) msg.send(fail_silently=False) except: transaction.rollback() raise else: transaction.commit() finally: transaction.leave_transaction_management() return self.user
def handle(self, *args, **options): if len(args) != 1: raise CommandError("Expected exactly 1 argument - Experiment ID") try: exp = Experiment.objects.get(pk=int(args[0])) except Experiment.DoesNotExist: raise CommandError("Experiment ID %s not found" % args[0]) self.stdout.write("Delete the following experiment?\n\n") # Print basic experiment information self.stdout.write("Experiment\n ID: {0}\n".format(exp.id)) self.stdout.write(" Title: {0}\n".format(exp.title)) self.stdout.write(" Public: {0}\n".format(exp.public)) # List experiment authors authors = Author_Experiment.objects.filter(experiment=exp) self.stdout.write(" Authors:\n") for author in authors: self.stdout.write(" {0}\n".format(author.author)) # List experiment metadata epsets = ExperimentParameterSet.objects.filter(experiment=exp) for epset in epsets: self.stdout.write(" Param Set: {0} - {1}\n".format( epset.schema.name, epset.schema.namespace)) params = ExperimentParameter.objects.filter(parameterset=epset) for param in params: self.stdout.write(" {0} = {1}\n".format( param.name.full_name, param.get())) # List experiment ACLs acls = ExperimentACL.objects.filter(experiment=exp) self.stdout.write(" ACLs:\n") for acl in acls: self.stdout.write(" {0}-{1}, flags: ".format( acl.pluginId, acl.entityId)) if acl.canRead: self.stdout.write("R") if acl.canWrite: self.stdout.write("W") if acl.canDelete: self.stdout.write("D") if acl.isOwner: self.stdout.write("O") self.stdout.write("\n") # Basic Statistics datasets = Dataset.objects.filter(experiment=exp) datafiles = Dataset_File.objects.filter(dataset__experiment=exp) self.stdout.write(" {0} datset(s), containing {1} file(s)\n".format( datasets.count(), datafiles.count())) # If the user has only requested a listing finish now if options.get('list', False): return # User must enter "yes" to proceed self.stdout.write("\n\nConfirm Deletion? (yes): ") ans = sys.stdin.readline().strip() if ans != "yes": self.stdout.write("'yes' not entered, aborting.\n") return # Consider the entire experiment deletion atomic using = options.get('database', DEFAULT_DB_ALIAS) transaction.commit_unless_managed(using=using) transaction.enter_transaction_management(using=using) transaction.managed(True, using=using) try: acls.delete() epsets.delete() DatasetParameterSet.objects.filter( dataset__experiment=exp).delete() DatafileParameterSet.objects.filter( dataset_file__dataset__experiment=exp).delete() authors.delete() datasets.delete() datafiles.delete() exp.delete() transaction.commit(using=using) transaction.leave_transaction_management(using=using) except: transaction.rollback(using=using) raise CommandError("Exception occurred, rolling back transaction")
def _post_teardown(self): transaction.rollback() transaction.leave_transaction_management()
def process_exception(self, request, exception): """Abort the transaction on errors.""" from django.db import transaction as django_transaction transaction.abort() django_transaction.set_clean() django_transaction.leave_transaction_management()
def delete_objects(seen_objs, using): """ Iterate through a list of seen classes, and remove any instances that are referred to. """ connection = connections[using] if not transaction.is_managed(using=using): transaction.enter_transaction_management(using=using) forced_managed = True else: forced_managed = False try: ordered_classes = seen_objs.keys() except CyclicDependency: # If there is a cyclic dependency, we cannot in general delete the # objects. However, if an appropriate transaction is set up, or if the # database is lax enough, it will succeed. So for now, we go ahead and # try anyway. ordered_classes = seen_objs.unordered_keys() obj_pairs = {} try: for cls in ordered_classes: items = seen_objs[cls].items() items.sort() obj_pairs[cls] = items # Pre-notify all instances to be deleted. for pk_val, instance in items: if not cls._meta.auto_created: signals.pre_delete.send(sender=cls, instance=instance) pk_list = [pk for pk,instance in items] del_query = sql.DeleteQuery(cls) del_query.delete_batch_related(pk_list, using=using) update_query = sql.UpdateQuery(cls) for field, model in cls._meta.get_fields_with_model(): if (field.rel and field.null and field.rel.to in seen_objs and filter(lambda f: f.column == field.rel.get_related_field().column, field.rel.to._meta.fields)): if model: sql.UpdateQuery(model).clear_related(field, pk_list, using=using) else: update_query.clear_related(field, pk_list, using=using) # Now delete the actual data. for cls in ordered_classes: items = obj_pairs[cls] items.reverse() pk_list = [pk for pk,instance in items] del_query = sql.DeleteQuery(cls) del_query.delete_batch(pk_list, using=using) # Last cleanup; set NULLs where there once was a reference to the # object, NULL the primary key of the found objects, and perform # post-notification. for pk_val, instance in items: for field in cls._meta.fields: if field.rel and field.null and field.rel.to in seen_objs: setattr(instance, field.attname, None) if not cls._meta.auto_created: signals.post_delete.send(sender=cls, instance=instance) setattr(instance, cls._meta.pk.attname, None) if forced_managed: transaction.commit(using=using) else: transaction.commit_unless_managed(using=using) finally: if forced_managed: transaction.leave_transaction_management(using=using)
def leave(self): if db_transaction.is_dirty(): db_transaction.rollback() db_transaction.leave_transaction_management(using=self.using)
def import_data(self, dataset, dry_run=False, raise_errors=False, use_transactions=None): """ Imports data from ``dataset``. ``use_transactions`` If ``True`` import process will be processed inside transaction. If ``dry_run`` is set, or error occurs, transaction will be rolled back. """ result = Result() result.diff_headers = self.get_diff_headers() if use_transactions is None: use_transactions = self.get_use_transactions() if use_transactions is True: # when transactions are used we want to create/update/delete object # as transaction will be rolled back if dry_run is set real_dry_run = False transaction.enter_transaction_management() transaction.managed(True) else: real_dry_run = dry_run instance_loader = self._meta.instance_loader_class(self, dataset) try: self.before_import(dataset, real_dry_run) except Exception as e: tb_info = traceback.format_exc(2) result.base_errors.append(Error(repr(e), tb_info)) if raise_errors: if use_transactions: transaction.rollback() transaction.leave_transaction_management() raise for row in dataset.dict: try: row_result = RowResult() instance, new = self.get_or_init_instance(instance_loader, row) if new: row_result.import_type = RowResult.IMPORT_TYPE_NEW else: row_result.import_type = RowResult.IMPORT_TYPE_UPDATE row_result.new_record = new original = deepcopy(instance) if self.for_delete(row, instance): if new: row_result.import_type = RowResult.IMPORT_TYPE_SKIP row_result.diff = self.get_diff(None, None, real_dry_run) else: row_result.import_type = RowResult.IMPORT_TYPE_DELETE self.delete_instance(instance, real_dry_run) row_result.diff = self.get_diff(original, None, real_dry_run) else: self.import_obj(instance, row, real_dry_run) if self.skip_row(instance, original): row_result.import_type = RowResult.IMPORT_TYPE_SKIP else: self.save_instance(instance, real_dry_run) self.save_m2m(instance, row, real_dry_run) # Add object info to RowResult for LogEntry row_result.object_repr = force_text(instance) row_result.object_id = instance.pk row_result.diff = self.get_diff(original, instance, real_dry_run) except Exception as e: tb_info = traceback.format_exc(2) row_result.errors.append(Error(e, tb_info)) if raise_errors: if use_transactions: transaction.rollback() transaction.leave_transaction_management() six.reraise(*sys.exc_info()) if (row_result.import_type != RowResult.IMPORT_TYPE_SKIP or self._meta.report_skipped): result.rows.append(row_result) if use_transactions: if dry_run or result.has_errors(): transaction.rollback() else: transaction.commit() transaction.leave_transaction_management() return result
model = obj.object.__class__ if router.allow_syncdb(using, model): models.add(model) counter += 1 obj.save(using=using) if counter > 0: sequence_sql = connection.ops.sequence_reset_sql(style, models) if sequence_sql: for line in sequence_sql: cursor.execute(line) except Exception, e: transaction.rollback(using=using) transaction.leave_transaction_management(using=using) raise e transaction.commit(using=using) transaction.leave_transaction_management(using=using) connection.close() return counter ########NEW FILE######## __FILENAME__ = views # Copyright (c) 2009 Guilherme Gondim and contributors # # This file is part of Django Smuggler. # # Django Smuggler is free software under terms of the GNU Lesser # General Public License version 3 (LGPLv3) as published by the Free # Software Foundation. See the file README for copying conditions. import os from datetime import datetime
def nocommit(using=None): t.enter_transaction_management(using=using) t.managed(True, using=using) yield t.rollback() t.leave_transaction_management(using=using)
def commit_work(run_transaction): if run_transaction: transaction.commit() transaction.leave_transaction_management() printv('Transaction committed')
def handle(self, *fixture_labels, **options): ignore = options.get('ignore') using = options.get('database') connection = connections[using] if not len(fixture_labels): raise CommandError( "No database fixture specified. Please provide the path of at " "least one fixture in the command line.") verbosity = int(options.get('verbosity')) show_traceback = options.get('traceback') # commit is a stealth option - it isn't really useful as # a command line option, but it can be useful when invoking # loaddata from within another script. # If commit=True, loaddata will use its own transaction; # if commit=False, the data load SQL will become part of # the transaction in place when loaddata was invoked. commit = options.get('commit', True) # Keep a count of the installed objects and fixtures fixture_count = 0 loaded_object_count = 0 fixture_object_count = 0 models = set() humanize = lambda dirname: "'%s'" % dirname if dirname else 'absolute path' # Get a cursor (even though we don't need one yet). This has # the side effect of initializing the test database (if # it isn't already initialized). cursor = connection.cursor() # Start transaction management. All fixtures are installed in a # single transaction to ensure that all references are resolved. if commit: transaction.commit_unless_managed(using=using) transaction.enter_transaction_management(using=using) transaction.managed(True, using=using) class SingleZipReader(zipfile.ZipFile): def __init__(self, *args, **kwargs): zipfile.ZipFile.__init__(self, *args, **kwargs) if settings.DEBUG: assert len( self.namelist() ) == 1, "Zip-compressed fixtures must contain only one file." def read(self): return zipfile.ZipFile.read(self, self.namelist()[0]) compression_types = { None: open, 'gz': gzip.GzipFile, 'zip': SingleZipReader } if has_bz2: compression_types['bz2'] = bz2.BZ2File app_module_paths = [] for app in get_apps(): if hasattr(app, '__path__'): # It's a 'models/' subpackage for path in app.__path__: app_module_paths.append(upath(path)) else: # It's a models.py module app_module_paths.append(upath(app.__file__)) app_fixtures = [ os.path.join(os.path.dirname(path), 'fixtures') for path in app_module_paths ] try: with connection.constraint_checks_disabled(): for fixture_label in fixture_labels: parts = fixture_label.split('.') if len(parts) > 1 and parts[-1] in compression_types: compression_formats = [parts[-1]] parts = parts[:-1] else: compression_formats = compression_types.keys() if len(parts) == 1: fixture_name = parts[0] formats = serializers.get_public_serializer_formats() else: fixture_name, format = '.'.join(parts[:-1]), parts[-1] if format in serializers.get_public_serializer_formats( ): formats = [format] else: formats = [] if formats: if verbosity >= 2: self.stdout.write("Loading '%s' fixtures..." % fixture_name) else: raise CommandError( "Problem installing fixture '%s': %s is not a known serialization format." % (fixture_name, format)) if os.path.isabs(fixture_name): fixture_dirs = [fixture_name] else: fixture_dirs = app_fixtures + list( settings.FIXTURE_DIRS) + [''] for fixture_dir in fixture_dirs: if verbosity >= 2: self.stdout.write("Checking %s for fixtures..." % humanize(fixture_dir)) label_found = False for combo in product([using, None], formats, compression_formats): database, format, compression_format = combo file_name = '.'.join(p for p in [ fixture_name, database, format, compression_format ] if p) if verbosity >= 3: self.stdout.write("Trying %s for %s fixture '%s'..." % \ (humanize(fixture_dir), file_name, fixture_name)) full_path = os.path.join(fixture_dir, file_name) open_method = compression_types[compression_format] try: fixture = open_method(full_path, 'r') except IOError: if verbosity >= 2: self.stdout.write("No %s fixture '%s' in %s." % \ (format, fixture_name, humanize(fixture_dir))) else: try: if label_found: raise CommandError( "Multiple fixtures named '%s' in %s. Aborting." % (fixture_name, humanize(fixture_dir))) fixture_count += 1 objects_in_fixture = 0 loaded_objects_in_fixture = 0 if verbosity >= 2: self.stdout.write("Installing %s fixture '%s' from %s." % \ (format, fixture_name, humanize(fixture_dir))) objects = serializers.deserialize( format, fixture, using=using, ignorenonexistent=ignore) for obj in objects: try: # Attempt to lookup any existing object using natural keys and # use that object's PK to duplicate and conflict records aren't created. nk = obj.object.natural_key() real_object = type( obj.object ).objects.get_by_natural_key(*nk) if real_object: obj.object.pk = real_object.pk except AttributeError: # Model class doesn't support natural keys. pass except type(obj.object).DoesNotExist: # No existing record, so proceed as normal. pass objects_in_fixture += 1 if router.allow_syncdb( using, obj.object.__class__): loaded_objects_in_fixture += 1 models.add(obj.object.__class__) try: obj.save(using=using) except (DatabaseError, IntegrityError) as e: e.args = ( "Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % { 'app_label': obj.object._meta. app_label, 'object_name': obj.object._meta. object_name, 'pk': obj.object.pk, 'error_msg': force_text(e) }, ) raise loaded_object_count += loaded_objects_in_fixture fixture_object_count += objects_in_fixture label_found = True except Exception as e: if not isinstance(e, CommandError): e.args = ( "Problem installing fixture '%s': %s" % (full_path, e), ) raise finally: fixture.close() # If the fixture we loaded contains 0 objects, assume that an # error was encountered during fixture loading. if objects_in_fixture == 0: raise CommandError( "No fixture data found for '%s'. (File format may be invalid.)" % (fixture_name)) # Since we disabled constraint checks, we must manually check for # any invalid keys that might have been added table_names = [model._meta.db_table for model in models] try: connection.check_constraints(table_names=table_names) except Exception as e: e.args = ("Problem installing fixtures: %s" % e, ) raise except (SystemExit, KeyboardInterrupt): raise except Exception as e: if commit: transaction.rollback(using=using) transaction.leave_transaction_management(using=using) raise # If we found even one object in a fixture, we need to reset the # database sequences. if loaded_object_count > 0: sequence_sql = connection.ops.sequence_reset_sql( no_style(), models) if sequence_sql: if verbosity >= 2: self.stdout.write("Resetting sequences\n") for line in sequence_sql: cursor.execute(line) if commit: transaction.commit(using=using) transaction.leave_transaction_management(using=using) if verbosity >= 1: if fixture_object_count == loaded_object_count: self.stdout.write("Installed %d object(s) from %d fixture(s)" % (loaded_object_count, fixture_count)) else: self.stdout.write( "Installed %d object(s) (of %d) from %d fixture(s)" % (loaded_object_count, fixture_object_count, fixture_count)) # Close the DB connection. This is required as a workaround for an # edge case in MySQL: if the same connection is used to # create tables, load data, and query, the query can return # incorrect results. See Django #7572, MySQL #37735. if commit: connection.close()
def handle(self, *fixture_labels, **options): from django.db.models import get_apps from django.core import serializers from django.db import connection, transaction from django.conf import settings self.style = no_style() verbosity = int(options.get('verbosity', 1)) show_traceback = options.get('traceback', False) # commit is a stealth option - it isn't really useful as # a command line option, but it can be useful when invoking # loaddata from within another script. # If commit=True, loaddata will use its own transaction; # if commit=False, the data load SQL will become part of # the transaction in place when loaddata was invoked. commit = options.get('commit', True) # Keep a count of the installed objects and fixtures fixture_count = 0 object_count = 0 models = set() humanize = lambda dirname: dirname and "'%s'" % dirname or 'absolute path' # Get a cursor (even though we don't need one yet). This has # the side effect of initializing the test database (if # it isn't already initialized). cursor = connection.cursor() # Start transaction management. All fixtures are installed in a # single transaction to ensure that all references are resolved. if commit: transaction.commit_unless_managed() transaction.enter_transaction_management() transaction.managed(True) self.disable_forward_ref_checks(cursor) class SingleZipReader(zipfile.ZipFile): def __init__(self, *args, **kwargs): zipfile.ZipFile.__init__(self, *args, **kwargs) if settings.DEBUG: assert len( self.namelist() ) == 1, "Zip-compressed fixtures must contain only one file." def read(self): return zipfile.ZipFile.read(self, self.namelist()[0]) compression_types = { None: file, 'gz': gzip.GzipFile, 'zip': SingleZipReader } if has_bz2: compression_types['bz2'] = bz2.BZ2File app_fixtures = [ os.path.join(os.path.dirname(app.__file__), 'fixtures') for app in get_apps() ] for fixture_label in fixture_labels: parts = fixture_label.split('.') if len(parts) > 1 and parts[-1] in compression_types: compression_formats = [parts[-1]] parts = parts[:-1] else: compression_formats = compression_types.keys() if len(parts) == 1: fixture_name = parts[0] formats = serializers.get_public_serializer_formats() else: fixture_name, format = '.'.join(parts[:-1]), parts[-1] if format in serializers.get_public_serializer_formats(): formats = [format] else: formats = [] if formats: if verbosity > 1: print "Loading '%s' fixtures..." % fixture_name else: self.enable_forward_ref_checks(cursor) sys.stderr.write( self.style.ERROR( "Problem installing fixture '%s': %s is not a known serialization format." % (fixture_name, format))) transaction.rollback() transaction.leave_transaction_management() return if os.path.isabs(fixture_name): fixture_dirs = [fixture_name] else: fixture_dirs = app_fixtures + list( settings.FIXTURE_DIRS) + [''] for fixture_dir in fixture_dirs: if verbosity > 1: print "Checking %s for fixtures..." % humanize(fixture_dir) label_found = False for format in formats: for compression_format in compression_formats: if compression_format: file_name = '.'.join( [fixture_name, format, compression_format]) else: file_name = '.'.join([fixture_name, format]) if verbosity > 1: print "Trying %s for %s fixture '%s'..." % \ (humanize(fixture_dir), file_name, fixture_name) full_path = os.path.join(fixture_dir, file_name) open_method = compression_types[compression_format] try: fixture = open_method(full_path, 'r') if label_found: fixture.close() self.enable_forward_ref_checks(cursor) print self.style.ERROR( "Multiple fixtures named '%s' in %s. Aborting." % (fixture_name, humanize(fixture_dir))) transaction.rollback() transaction.leave_transaction_management() return else: fixture_count += 1 objects_in_fixture = 0 if verbosity > 0: print "Installing %s fixture '%s' from %s." % \ (format, fixture_name, humanize(fixture_dir)) try: objects = serializers.deserialize( format, fixture) for obj in objects: objects_in_fixture += 1 self.handle_ref_checks(cursor, obj) models.add(obj.object.__class__) obj.save() object_count += objects_in_fixture label_found = True except (SystemExit, KeyboardInterrupt): self.enable_forward_ref_checks(cursor) raise except Exception: import traceback fixture.close() self.enable_forward_ref_checks(cursor) transaction.rollback() transaction.leave_transaction_management() if show_traceback: traceback.print_exc() else: sys.stderr.write( self.style.ERROR( "Problem installing fixture '%s': %s\n" % (full_path, ''.join( traceback.format_exception( sys.exc_type, sys.exc_value, sys.exc_traceback))))) return fixture.close() # If the fixture we loaded contains 0 objects, assume that an # error was encountered during fixture loading. if objects_in_fixture == 0: self.enable_forward_ref_checks(cursor) sys.stderr.write( self.style.ERROR( "No fixture data found for '%s'. (File format may be invalid.)" % (fixture_name))) transaction.rollback() transaction.leave_transaction_management() return except Exception, e: if verbosity > 1: print "No %s fixture '%s' in %s." % \ (format, fixture_name, humanize(fixture_dir))
def setupBridge(master, settings, config): '''Setup the bridget between buildbot and the database. This is also the closure in which all things happen that depend on the given settings. ''' # allow settings to be none for tests if settings is not None: os.environ['DJANGO_SETTINGS_MODULE'] = settings import bb2mbdb.utils reload(bb2mbdb.utils) import mbdb.models reload(mbdb.models) from bb2mbdb.utils import modelForSource, modelForChange, modelForLog, \ timeHelper from mbdb.models import Master, Slave, Builder, BuildRequest, Build from django.db import transaction try: # hack around the lack of @transaction.commit_manually transaction.enter_transaction_management() transaction.managed(True) dbm, new_master = Master.objects.get_or_create(name=master) transaction.commit() except: transaction.rollback() raise finally: transaction.leave_transaction_management() class Scheduler(BaseScheduler): @transaction.commit_on_success def addChange(self, change): dbchange = modelForChange(dbm, change) log.msg('ADDED CHANGE to DB, %d' % dbchange.number) def listBuilderNames(self): # Sadly, we need this. Buildbot is going to complain that we # don't build. What does he know. return [] if 'schedulers' not in config: config['schedulers'] = [] config['schedulers'].insert(0, Scheduler('bb2mbdb')) class StepReceiver(StatusReceiver): '''Build- and StatusReceiver helper objects to receive all events for a particular step. ''' def __init__(self, dbstep, basedir): self.step = dbstep self.basedir = basedir @transaction.commit_on_success def stepTextChanged(self, build, step, text): self.step.text = text self.step.save() @transaction.commit_on_success def stepText2Changed(self, build, step, text2): self.step.text2 = text2 self.step.save() @transaction.commit_on_success def logStarted(self, build, step, log): self.log = modelForLog(self.step, log, self.basedir) def logChunk(self, build, step, log, channel, text): pass @transaction.commit_on_success def logFinished(self, build, step, log): self.log.isFinished = True self.log.save() pass def stepETAUpdate(self, build, step, ETA, expectations): '''TODO: ETA support. ''' pass class BuildReceiver(StatusReceiver): '''StatusReceiver helper object to receive all events for a particular build. Caches the database model object. ''' def __init__(self, dbbuild, basedir): self.build = dbbuild self.basedir = basedir self.latestStep = self.latestDbStep = None @transaction.commit_manually def stepStarted(self, build, step): self.latestStep = step starttime = timeHelper(step.getTimes()[0]) self.latestDbStep = self.build.steps.create(name=step.getName(), starttime=starttime, text=step.getText(), text2=step.text2) transaction.commit() return StepReceiver(self.latestDbStep, self.basedir) @transaction.commit_on_success def stepFinished(self, build, step, results): assert step == self.latestStep, "We lost a step somewhere" try: self.latestStep = None self.latestDbStep.endtime = timeHelper(step.getTimes()[1]) # only the first is the result, the second is text2, # ignore that. self.latestDbStep.result = results[0] self.latestDbStep.text = step.getText() self.latestDbStep.text2 = step.text2 self.latestDbStep.save() self.latestDbStep = None except Exception, e: log.msg(str(e)) pass def buildETAUpdate(self, build, ETA): '''TODO: ETA support. ''' pass
def import_data(self, dataset, dry_run=False, raise_errors=False, use_transactions=None): """ Imports data from ``dataset``. ``use_transactions`` If ``True`` import process will be processed inside transaction. If ``dry_run`` is set, or error occurs, transaction will be rolled back. """ result = Result() if use_transactions is None: use_transactions = self.get_use_transactions() if use_transactions is True: # when transactions are used we want to create/update/delete object # as transaction will be rolled back if dry_run is set real_dry_run = False transaction.enter_transaction_management() transaction.managed(True) else: real_dry_run = dry_run instance_loader = self._meta.instance_loader_class(self, dataset) for row in dataset.dict: try: row_result = RowResult() instance, new = self.get_or_init_instance(instance_loader, row) if new: row_result.import_type = RowResult.IMPORT_TYPE_NEW else: row_result.import_type = RowResult.IMPORT_TYPE_UPDATE row_result.new_record = new original = deepcopy(instance) if self.for_delete(row, instance): if new: row_result.import_type = RowResult.IMPORT_TYPE_SKIP row_result.diff = self.get_diff( None, None, real_dry_run) else: row_result.import_type = RowResult.IMPORT_TYPE_DELETE self.delete_instance(instance, real_dry_run) row_result.diff = self.get_diff( original, None, real_dry_run) else: self.import_obj(instance, row) self.save_instance(instance, real_dry_run) self.save_m2m(instance, row, real_dry_run) row_result.diff = self.get_diff(original, instance, real_dry_run) except Exception, e: tb_info = traceback.format_exc(sys.exc_info()[2]) row_result.errors.append(Error(repr(e), tb_info)) if raise_errors: if use_transactions: transaction.rollback() transaction.leave_transaction_management() raise result.rows.append(row_result)
class Resource(object): """ Resource defines how objects are mapped to their import and export representations and handle importing and exporting data. """ __metaclass__ = DeclarativeMetaclass def get_use_transactions(self): if self._meta.use_transactions is None: return USE_TRANSACTIONS else: return self._meta.use_transactions def get_fields(self): """ Returns fields in ``export_order`` order. """ return [self.fields[f] for f in self.get_export_order()] @classmethod def get_field_name(cls, field): """ Returns field name for given field. """ for field_name, f in cls.fields.items(): if f == field: return field_name raise AttributeError("Field %s does not exists in %s resource" % (field, cls)) def init_instance(self, row=None): raise NotImplementedError() def get_instance(self, instance_loader, row): return instance_loader.get_instance(row) def get_or_init_instance(self, instance_loader, row): instance = self.get_instance(instance_loader, row) if instance: return (instance, False) else: return (self.init_instance(row), True) def save_instance(self, instance, dry_run=False): self.before_save_instance(instance, dry_run) if not dry_run: instance.save() self.after_save_instance(instance, dry_run) def before_save_instance(self, instance, dry_run): """ Override to add additional logic. """ pass def after_save_instance(self, instance, dry_run): """ Override to add additional logic. """ pass def delete_instance(self, instance, dry_run=False): self.before_delete_instance(instance, dry_run) if not dry_run: instance.delete() self.after_delete_instance(instance, dry_run) def before_delete_instance(self, instance, dry_run): """ Override to add additional logic. """ pass def after_delete_instance(self, instance, dry_run): """ Override to add additional logic. """ pass def import_field(self, field, obj, data): if field.attribute and field.column_name in data: field.save(obj, data) def import_obj(self, obj, data): """ """ for field in self.get_fields(): if isinstance(field.widget, widgets.ManyToManyWidget): continue self.import_field(field, obj, data) def save_m2m(self, obj, data, dry_run): """ Saves m2m fields. Model instance need to have a primary key value before a many-to-many relationship can be used. """ if not dry_run: for field in self.get_fields(): if not isinstance(field.widget, widgets.ManyToManyWidget): continue self.import_field(field, obj, data) def for_delete(self, row, instance): """ Returns ``True`` if ``row`` importing should delete instance. Default implementation returns ``False``. Override this method to handle deletion. """ return False def get_diff(self, original, current, dry_run=False): """ Get diff between original and current object when ``import_data`` is run. ``dry_run`` allows handling special cases when object is not saved to database (ie. m2m relationships). """ data = [] dmp = diff_match_patch() for field in self.get_fields(): v1 = self.export_field(field, original) if original else "" v2 = self.export_field(field, current) if current else "" diff = dmp.diff_main(unicode(v1), unicode(v2)) dmp.diff_cleanupSemantic(diff) html = dmp.diff_prettyHtml(diff) html = mark_safe(html) data.append(html) return data def get_diff_headers(self): """ Diff representation headers. """ return self.get_export_headers() def import_data(self, dataset, dry_run=False, raise_errors=False, use_transactions=None): """ Imports data from ``dataset``. ``use_transactions`` If ``True`` import process will be processed inside transaction. If ``dry_run`` is set, or error occurs, transaction will be rolled back. """ result = Result() if use_transactions is None: use_transactions = self.get_use_transactions() if use_transactions is True: # when transactions are used we want to create/update/delete object # as transaction will be rolled back if dry_run is set real_dry_run = False transaction.enter_transaction_management() transaction.managed(True) else: real_dry_run = dry_run instance_loader = self._meta.instance_loader_class(self, dataset) for row in dataset.dict: try: row_result = RowResult() instance, new = self.get_or_init_instance(instance_loader, row) if new: row_result.import_type = RowResult.IMPORT_TYPE_NEW else: row_result.import_type = RowResult.IMPORT_TYPE_UPDATE row_result.new_record = new original = deepcopy(instance) if self.for_delete(row, instance): if new: row_result.import_type = RowResult.IMPORT_TYPE_SKIP row_result.diff = self.get_diff( None, None, real_dry_run) else: row_result.import_type = RowResult.IMPORT_TYPE_DELETE self.delete_instance(instance, real_dry_run) row_result.diff = self.get_diff( original, None, real_dry_run) else: self.import_obj(instance, row) self.save_instance(instance, real_dry_run) self.save_m2m(instance, row, real_dry_run) row_result.diff = self.get_diff(original, instance, real_dry_run) except Exception, e: tb_info = traceback.format_exc(sys.exc_info()[2]) row_result.errors.append(Error(repr(e), tb_info)) if raise_errors: if use_transactions: transaction.rollback() transaction.leave_transaction_management() raise result.rows.append(row_result) if use_transactions: if dry_run or result.has_errors(): transaction.rollback() else: transaction.commit() transaction.leave_transaction_management() return result
def process_exception(self, request, exception): """Rolls back the database and leaves transaction management""" if transaction.is_dirty(): transaction.rollback() transaction.leave_transaction_management()
def tearDown(self): # Close down the second connection. transaction.leave_transaction_management() self.conn2.close()
def handle(self, *args, **options): if len(args) != 1: raise CommandError("Expected exactly 1 argument - Experiment ID") try: exp = Experiment.objects.get(pk=int(args[0])) except Experiment.DoesNotExist: raise CommandError("Experiment ID %s not found" % args[0]) # FIXME - we are fetch a bunch of stuff outside of any transaction, and then # doing the deletes in a transaction. There is an obvious race condition here # that may result in components of an experiment not being deleted or being deleted # when they shouldn't be. # Fetch Datasets and Datafiles and work out which ones would be deleted datasets = Dataset.objects.filter(experiments__id=exp.id) datafiles = Dataset_File.objects.filter( dataset__id__in=map((lambda ds: ds.id), datasets)) uniqueDatasets = filter((lambda ds: ds.experiments.count() == 1), datasets) uniqueDatasetIds = map((lambda ds: ds.id), uniqueDatasets) uniqueDatafiles = filter( (lambda df: df.dataset.id in uniqueDatasetIds), datafiles) # Fetch other stuff to be printed and deleted. acls = ExperimentACL.objects.filter(experiment=exp) authors = Author_Experiment.objects.filter(experiment=exp) epsets = ExperimentParameterSet.objects.filter(experiment=exp) confirmed = options.get('confirmed', False) listOnly = options.get('list', False) if not listOnly and not confirmed: self.stdout.write("Delete the following experiment?\n\n") if listOnly or not confirmed: # Print basic experiment information self.stdout.write("Experiment\n ID: {0}\n".format(exp.id)) self.stdout.write(" Title: {0}\n".format(exp.title)) self.stdout.write(" Locked: {0}\n".format(exp.locked)) self.stdout.write(" Public Access: {0}\n".format( exp.public_access)) # List experiment authors self.stdout.write(" Authors:\n") for author in authors: self.stdout.write(" {0}\n".format(author.author)) # List experiment metadata for epset in epsets: self.stdout.write(" Param Set: {0} - {1}\n".format( epset.schema.name, epset.schema.namespace)) params = ExperimentParameter.objects.filter(parameterset=epset) for param in params: self.stdout.write(" {0} = {1}\n".format( param.name.full_name, param.get())) # List experiment ACLs self.stdout.write(" ACLs:\n") for acl in acls: self.stdout.write(" {0}-{1}, flags: ".format( acl.pluginId, acl.entityId)) if acl.canRead: self.stdout.write("R") if acl.canWrite: self.stdout.write("W") if acl.canDelete: self.stdout.write("D") if acl.isOwner: self.stdout.write("O") self.stdout.write("\n") # Basic Statistics self.stdout.write( " {0} total dataset(s), containing {1} file(s)\n".format( datasets.count(), datafiles.count())) self.stdout.write( " {0} non-shared dataset(s), containing {1} file(s)\n". format(len(uniqueDatasets), len(uniqueDatafiles))) if len(uniqueDatasets) > 0 and not listOnly: self.stdout.write( " (The non-shared datasets and files will be deleted)\n" ) # If the user has only requested a listing finish now if listOnly: return if not confirmed: # User must enter "yes" to proceed self.stdout.write("\n\nConfirm Deletion? (yes): ") ans = sys.stdin.readline().strip() if ans != "yes": self.stdout.write("'yes' not entered, aborting.\n") return # Consider the entire experiment deletion atomic using = options.get('database', DEFAULT_DB_ALIAS) transaction.commit_unless_managed(using=using) transaction.enter_transaction_management(using=using) transaction.managed(True, using=using) try: acls.delete() epsets.delete() for dataset in datasets: dataset.experiments.remove(exp.id) if dataset.experiments.count() == 0: DatasetParameterSet.objects.filter( dataset=dataset).delete() for datafile in Dataset_File.objects.filter( dataset=dataset): DatafileParameterSet.objects.filter( dataset_file=datafile).delete() datafile.delete() dataset.delete() authors.delete() exp.delete() transaction.commit(using=using) transaction.leave_transaction_management(using=using) except Exception: transaction.rollback(using=using) exc_class, exc, tb = sys.exc_info() new_exc = CommandError( "Exception %s has occurred: rolled back transaction" % (exc or exc_class)) raise new_exc.__class__, new_exc, tb
elif line[0] != "#": sys.stderr.write("Junk data on line %s" % i) db.reset_queries() i += 1 pct = (i * 100 / totalobjs) if pct != prev_pct: sys.stdout.write(" [%s%%]\r" % pct) sys.stdout.flush() prev_pct = pct f.close() transaction.commit() transaction.leave_transaction_management() else: # Legacy dumpdata output. Try loading it directly. print "Importing old style dump format. This may take a while." f.close() os.system("./reviewboard/manage.py loaddata %s" % filename) except Exception, e: f.close() sys.stderr.write("Problem installing '%s': %s\n" % (filename, str(e))) sys.exit(1) if transaction_setup: transaction.rollback() transaction.leave_transaction_management() print
def migrate_model(processor, model, fields): from ella.core.models import Publishable model = get_model(*model.split('.')) ct = ContentType.objects.get_for_model(model) if model == Publishable: ct = None print 'processing', model._meta, ':', sys.stdout.flush() converted = 0 deps = 0 try: enter_transaction_management() managed(True) try: for m in model.objects.order_by().iterator(): if not ct: # publishable ct = ContentType.objects.get_for_id(m.content_type_id) sys.stdout.write('.') converted += 1 # commit every 1000 iterations if (converted % 1000) == 0 and is_dirty(): commit() sys.stdout.write('C') sys.stdout.flush() dirty = False for f in fields: val = getattr(m, f) if val: val, cnt = BOX_RE.subn(update_field(m, ct), val) if cnt > 0: deps += cnt setattr(m, f, val) dirty = True SourceText.objects.extract_from_instance(m, processor, fields, content_type=ct, force_save=dirty, force_create=True) except: # rollback and propagate if something goes wrong if is_dirty(): rollback() raise else: # commit at the end if is_dirty(): commit() finally: leave_transaction_management() print print 'DONE converted %d (%d reported dependencies)' % ( converted, deps, ) sys.stdout.flush()
def handle(self, *fixture_files, **options): using = options.get('database', DEFAULT_DB_ALIAS) mode = options.get('mode', 'append') items_into_tree = options.get('into_tree', None) if items_into_tree is not None: try: items_into_tree = Tree.objects.get(alias=items_into_tree) except ObjectDoesNotExist: raise CommandError( 'Target tree alised by `%s` does not exist. Please create it before import.' % items_into_tree) else: mode = 'append' connection = connections[using] cursor = connection.cursor() self.style = no_style() transaction.commit_unless_managed(using=using) transaction.enter_transaction_management(using=using) transaction.managed(True, using=using) loaded_object_count = 0 if mode == 'replace': try: Tree.objects.all().delete() TreeItem.objects.all().delete() except ObjectDoesNotExist: pass for fixture_file in fixture_files: self.stdout.write('Loading fixture from `%s` ...\n' % fixture_file) fixture = file(fixture_file, 'r') try: objects = serializers.deserialize('json', fixture, using=using) except (SystemExit, KeyboardInterrupt): raise trees = [] tree_items = defaultdict(list) tree_item_parents = defaultdict(list) tree_items_new_indexes = {} for obj in objects: if router.allow_syncdb(using, obj.object.__class__): if isinstance(obj.object, (Tree, TreeItem)): if isinstance(obj.object, Tree): trees.append(obj.object) else: if items_into_tree is not None: obj.object.tree_id = items_into_tree.id tree_items[obj.object.tree_id].append(obj.object) tree_item_parents[obj.object.parent_id].append( obj.object.id) if items_into_tree is not None: trees = [ items_into_tree, ] try: for tree in trees: self.stdout.write('\nImporting tree `%s` ...\n' % tree.alias) orig_tree_id = tree.id if items_into_tree is None: if mode == 'append': tree.pk = None tree.id = None tree.save(using=using) loaded_object_count += 1 parents_ahead = [] for tree_item in tree_items[orig_tree_id]: parent_ahead = False self.stdout.write('Importing item `%s` ...\n' % tree_item.title) tree_item.tree_id = tree.id orig_item_id = tree_item.id if mode == 'append': tree_item.pk = None tree_item.id = None if tree_item.id in tree_items_new_indexes: tree_item.pk = tree_item.id = tree_items_new_indexes[ tree_item.id] if tree_item.parent_id is not None: if tree_item.parent_id in tree_items_new_indexes: tree_item.parent_id = tree_items_new_indexes[ tree_item.parent_id] else: parent_ahead = True tree_item.save(using=using) loaded_object_count += 1 if mode == 'append': tree_items_new_indexes[orig_item_id] = tree_item.id if parent_ahead: parents_ahead.append(tree_item) # Second pass is necessary for tree items being imported before their parents. for tree_item in parents_ahead: tree_item.parent_id = tree_items_new_indexes[ tree_item.parent_id] tree_item.save(using=using) except (SystemExit, KeyboardInterrupt): raise except Exception: import traceback fixture.close() transaction.rollback(using=using) transaction.leave_transaction_management(using=using) self.stderr.write( self.style.ERROR('Fixture `%s` import error: %s\n' % (fixture_file, ''.join( traceback.format_exception( sys.exc_type, sys.exc_value, sys.exc_traceback))))) fixture.close() # Reset DB sequences, for DBMS with sequences support. if loaded_object_count > 0: sequence_sql = connection.ops.sequence_reset_sql( self.style, [Tree, TreeItem]) if sequence_sql: self.stdout.write('Resetting DB sequences ...\n') for line in sequence_sql: cursor.execute(line) transaction.commit(using=using) transaction.leave_transaction_management(using=using) connection.close()
def _leave_transaction_management(self): transaction.leave_transaction_management(using=self.using) if not connections[self.using].is_managed() and \ connections[self.using].features.uses_autocommit: connections[self.using]._set_isolation_level( psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
def handle(self, *fixture_labels, **options): from django.db.models import get_apps from django.core import serializers from django.db import connection, transaction from django.conf import settings self.style = no_style() verbosity = int(options.get('verbosity', 1)) show_traceback = options.get('traceback', False) # commit is a stealth option - it isn't really useful as # a command line option, but it can be useful when invoking # loaddata from within another script. # If commit=True, loaddata will use its own transaction; # if commit=False, the data load SQL will become part of # the transaction in place when loaddata was invoked. commit = options.get('commit', True) # Keep a count of the installed objects and fixtures fixture_count = 0 object_count = 0 objects_per_fixture = [] models = set() humanize = lambda dirname: dirname and "'%s'" % dirname or 'absolute path' # Get a cursor (even though we don't need one yet). This has # the side effect of initializing the test database (if # it isn't already initialized). cursor = connection.cursor() # Start transaction management. All fixtures are installed in a # single transaction to ensure that all references are resolved. if commit: transaction.commit_unless_managed() transaction.enter_transaction_management() transaction.managed(True) app_fixtures = [ os.path.join(os.path.dirname(app.__file__), 'fixtures') for app in get_apps() ] for fixture_label in fixture_labels: parts = fixture_label.split('.') if len(parts) == 1: fixture_name = fixture_label formats = serializers.get_public_serializer_formats() else: fixture_name, format = '.'.join(parts[:-1]), parts[-1] if format in serializers.get_public_serializer_formats(): formats = [format] else: formats = [] if formats: if verbosity > 1: print "Loading '%s' fixtures..." % fixture_name else: sys.stderr.write( self.style.ERROR( "Problem installing fixture '%s': %s is not a known serialization format." % (fixture_name, format))) transaction.rollback() transaction.leave_transaction_management() return if os.path.isabs(fixture_name): fixture_dirs = [fixture_name] else: fixture_dirs = app_fixtures + list( settings.FIXTURE_DIRS) + [''] for fixture_dir in fixture_dirs: if verbosity > 1: print "Checking %s for fixtures..." % humanize(fixture_dir) label_found = False for format in formats: serializer = serializers.get_serializer(format) if verbosity > 1: print "Trying %s for %s fixture '%s'..." % \ (humanize(fixture_dir), format, fixture_name) try: full_path = os.path.join( fixture_dir, '.'.join([fixture_name, format])) fixture = open(full_path, 'r') if label_found: fixture.close() print self.style.ERROR( "Multiple fixtures named '%s' in %s. Aborting." % (fixture_name, humanize(fixture_dir))) transaction.rollback() transaction.leave_transaction_management() return else: fixture_count += 1 objects_per_fixture.append(0) if verbosity > 0: print "Installing %s fixture '%s' from %s." % \ (format, fixture_name, humanize(fixture_dir)) try: objects = serializers.deserialize( format, fixture) for obj in objects: object_count += 1 objects_per_fixture[-1] += 1 models.add(obj.object.__class__) obj.save() label_found = True except (SystemExit, KeyboardInterrupt): raise except Exception: import traceback fixture.close() transaction.rollback() transaction.leave_transaction_management() if show_traceback: import traceback traceback.print_exc() else: sys.stderr.write( self.style.ERROR( "Problem installing fixture '%s': %s\n" % (full_path, traceback.format_exc()))) return fixture.close() except: if verbosity > 1: print "No %s fixture '%s' in %s." % \ (format, fixture_name, humanize(fixture_dir)) # If any of the fixtures we loaded contain 0 objects, assume that an # error was encountered during fixture loading. if 0 in objects_per_fixture: sys.stderr.write( self.style.ERROR( "No fixture data found for '%s'. (File format may be invalid.)" % (fixture_name))) transaction.rollback() transaction.leave_transaction_management() return # If we found even one object in a fixture, we need to reset the # database sequences. if object_count > 0: sequence_sql = connection.ops.sequence_reset_sql( self.style, models) if sequence_sql: if verbosity > 1: print "Resetting sequences" for line in sequence_sql: cursor.execute(line) if commit: transaction.commit() transaction.leave_transaction_management() if object_count == 0: if verbosity > 1: print "No fixtures found." else: if verbosity > 0: print "Installed %d object(s) from %d fixture(s)" % ( object_count, fixture_count) # Close the DB connection. This is required as a workaround for an # edge case in MySQL: if the same connection is used to # create tables, load data, and query, the query can return # incorrect results. See Django #7572, MySQL #37735. if commit: connection.close()
def update_imported_docs(project, version): """ Check out or update the given project's repository. """ update_docs_output = {} if not project.vcs_repo(): raise ProjectImportError("Repo type '{repo_type}' unknown".format( repo_type=project.repo_type)) if version: log.info('Checking out version {slug}: {identifier}'.format( slug=version.slug, identifier=version.identifier)) version_slug = version.slug version_repo = project.vcs_repo(version_slug) update_docs_output['checkout'] = version_repo.checkout( version.identifier) else: log.info('Updating to latest revision') version_slug = 'latest' version_repo = project.vcs_repo(version_slug) update_docs_output['checkout'] = version_repo.update() # Ensure we have a conf file (an exception is raised if not) conf_file = project.conf_file(version.slug) #Do Virtualenv bits: if project.use_virtualenv: update_docs_output['venv'] = run( '{cmd} --distribute --never-download --no-site-packages {path}'. format(cmd='virtualenv', path=project.venv_path(version=version_slug))) update_docs_output['sphinx'] = run('{cmd} install -U sphinx'.format( cmd=project.venv_bin(version=version_slug, bin='pip'))) if project.requirements_file: os.chdir(project.checkout_path(version_slug)) update_docs_output['requirements'] = run( '{cmd} install -r {requirements}'.format( cmd=project.venv_bin(version=version_slug, bin='pip'), requirements=project.requirements_file)) os.chdir(project.checkout_path(version_slug)) update_docs_output['install'] = run( '{cmd} setup.py install --force'.format( cmd=project.venv_bin(version=version_slug, bin='python'))) # check tags/version #XXX:dc: what in this block raises the values error? try: if version_repo.supports_tags: transaction.enter_transaction_management(True) tags = version_repo.tags old_tags = [ obj['identifier'] for obj in api.version.get(project__slug=project.slug, limit=50)['objects'] ] for tag in tags: if tag.identifier in old_tags: continue slug = slugify_uniquely(Version, tag.verbose_name, 'slug', 255, project=project) try: api.version.post( dict(project="/api/v1/project/%s/" % project.pk, slug=slug, identifier=tag.identifier, verbose_name=tag.verbose_name)) log.info("New tag found: {0}".format(tag.identifier)) highest = project.highest_version['version'] ver_obj = mkversion(ver) #TODO: Handle updating higher versions automatically. #This never worked very well, anyways. if highest and ver_obj and ver_obj > highest: log.info("Highest verison known, building docs") update_docs.delay(ver.project.pk, version_pk=ver.pk) except Exception, e: log.error("Failed to create version (tag)", exc_info=True) transaction.rollback() #break transaction.leave_transaction_management() if version_repo.supports_branches: transaction.enter_transaction_management(True) branches = version_repo.branches old_branches = [ obj['identifier'] for obj in api.version.get(project__slug=project.slug, limit=50)['objects'] ] for branch in branches: if branch.identifier in old_branches: continue slug = slugify_uniquely(Version, branch.verbose_name, 'slug', 255, project=project) try: api.version.post( dict(project="/api/v1/project/%s/" % project.pk, slug=slug, identifier=branch.identifier, verbose_name=branch.verbose_name)) log.info("New branch found: {0}".format(branch.identifier)) except Exception, e: log.error("Failed to create version (branch)", exc_info=True) transaction.rollback() #break transaction.leave_transaction_management()
e.doAction() # Now update the evolution table version = Version(signature=current_signature) version.save(**using_args) for evolution in new_evolutions: evolution.version = version evolution.save(**using_args) transaction.commit(**using_args) except Exception, ex: transaction.rollback(**using_args) raise CommandError('Error applying evolution: %s' % str(ex)) transaction.leave_transaction_management(**using_args) if verbosity > 0: print 'Evolution successful.' else: print self.style.ERROR('Evolution cancelled.') elif not compile_sql: if verbosity > 0: if simulated: print "Trial evolution successful." print "Run './manage.py evolve %s--execute' to apply evolution." % ( hint and '--hint ' or '') elif verbosity > 0: print 'No evolution required.'
def tearDown(self): # The tests access the database after exercising 'atomic', making the # connection dirty; a rollback is required to make it clean. transaction.rollback() transaction.leave_transaction_management()
def test_atomic_prevents_calling_transaction_management_methods(self): with transaction.atomic(): with self.assertRaises(transaction.TransactionManagementError): transaction.enter_transaction_management() with self.assertRaises(transaction.TransactionManagementError): transaction.leave_transaction_management()
def create_data(url_root): gateway = urljoin(url_root, 'router/') transaction.enter_transaction_management() doctor = User() doctor.mcid = mcid_generator.next_mcid() doctor.first_name = 'Demo' doctor.last_name = 'Doctor' doctor.email = '*****@*****.**' doctor.updatetime = 0 doctor.ccrlogupdatetime = 0 doctor.save() physician = User() physician.mcid = mcid_generator.next_mcid() physician.first_name = 'Demo' physician.last_name = 'Physician' physician.email = '*****@*****.**' physician.updatetime = 0 physician.ccrlogupdatetime = 0 physician.save() g, p = create_group('Demo Group Worklist', '*****@*****.**', url_root, doctor.mcid) add_to_group(g, physician.mcid) patients = [] for first_name, last_name, email, currentccr, reason, ccrs in PATIENTS: user = User() user.first_name = first_name user.last_name = last_name user.email = email user.mcid = mcid_generator.next_mcid() user.acctype = 'USER' user.rolehack = 'ccrlhm' user.updatetime = 0 user.ccrlogupdatetime = 0 user.save() patients.append(user) # Set worklist sql_execute(ACCOUNT_RLS_SQL, user.mcid, p.practiceRlsUrl) if not currentccr: continue sql_execute(DOCUMENT_TYPE_SQL, user.mcid, currentccr) ev = Practiceccrevent() ev.practiceid = p ev.PatientGivenName = user.first_name ev.PatientFamilyName = user.last_name ev.PatientIdentifier = user.mcid ev.PatientIdentifierSource = 'Patient Medcommons ID' ev.Guid = currentccr ev.Purpose = reason ev.SenderProviderId = 'idp' ev.ReceiverProviderId = 'idp' ev.DOB = '16 Jan 1968 05:00:00 GMT' ev.CXPServerURL = '' ev.CXPServerVendor = 'Medcommons' ev.ViewerURL = urljoin(gateway, 'access?g=%s' % currentccr) ev.Comment = '\n 3D Imaging Consult\n ' ev.CreationDateTime = 1162365858 ev.ConfirmationCode = tn_generator.next_tracking_number() ev.RegistrySecret = '' ev.PatientSex = 'Female' ev.PatientAge = '' ev.Status = 'New' ev.ViewStatus = 'Visible' ev.save() sql_execute(CCRLOG_SQL, user.mcid, currentccr, 'CCR', ev.ConfirmationCode) for ccr in ccrs: sql_execute(CCRLOG_SQL, user.mcid, ccr, 'CCR', tn_generator.next_tracking_number()) sql_execute(TODIR_SQL, g.groupinstanceid, doctor.email, doctor.email, doctor.mcid) sql_execute(TODIR_SQL, g.groupinstanceid, physician.email, physician.email, physician.mcid) demoCCR = 'fdfbbb9cf53f8577b420ed72567cd2104589fb0d' sql_execute(CCRLOG_SQL, doctor.mcid, demoCCR, 'DICOM Import', tn_generator.next_tracking_number()) sql_execute(CCRLOG_SQL, patients[0].mcid, demoCCR, 'DICOM Import', tn_generator.next_tracking_number()) sql_execute(CCRLOG_SQL, patients[0].mcid, PATIENTS[0][3], 'DICOM Import', tn_generator.next_tracking_number()) # Secondary group if 0: g2, p2 = create_group('Healthy Doctors', '*****@*****.**', url_root, doctor.mcid) p = MCProperty() p.property = 'acDemoDoctor' p.value = doctor.mcid p.save() transaction.leave_transaction_management()
def handle(self, **options): # Pick up the options if 'database' in options: database = options['database'] or DEFAULT_DB_ALIAS else: database = DEFAULT_DB_ALIAS if not database in settings.DATABASES: raise CommandError("No database settings known for '%s'" % database ) if 'user' in options and options['user']: try: user = User.objects.all().using(database).get(username=options['user']) except: raise CommandError("User '%s' not found" % options['user'] ) else: user = None now = datetime.now() transaction.enter_transaction_management(using=database) task = None try: # Initialize the task if 'task' in options and options['task']: try: task = Task.objects.all().using(database).get(pk=options['task']) except: raise CommandError("Task identifier not found") if task.started or task.finished or task.status != "Waiting" or task.name != 'generate plan': raise CommandError("Invalid task identifier") task.status = '0%' task.started = now else: task = Task(name='generate plan', submitted=now, started=now, status='0%', user=user) # Validate options if 'constraint' in options: constraint = int(options['constraint']) if constraint < 0 or constraint > 15: raise ValueError("Invalid constraint: %s" % options['constraint']) else: constraint = 15 if 'plantype' in options: plantype = int(options['plantype']) if plantype < 1 or plantype > 2: raise ValueError("Invalid plan type: %s" % options['plantype']) else: plantype = 1 if options['env']: task.arguments = "--constraint=%d --plantype=%d --env=%s" % (constraint, plantype, options['env']) for i in options['env'].split(','): j = i.split('=') if len(j) == 1: os.environ[j[0]] = '1' else: os.environ[j[0]] = j[1] else: task.arguments = "--constraint=%d --plantype=%d" % (constraint, plantype) # Log task task.save(using=database) transaction.commit(using=database) # Locate commands.py cmd = None for app in settings.INSTALLED_APPS: mod = import_module(app) if os.path.exists(os.path.join(os.path.dirname(mod.__file__),'commands.py')): cmd = os.path.join(os.path.dirname(mod.__file__),'commands.py') break if not cmd: raise Exception("Can't locate commands.py") # Execute os.environ['FREPPLE_PLANTYPE'] = str(plantype) os.environ['FREPPLE_CONSTRAINT'] = str(constraint) os.environ['FREPPLE_TASKID'] = str(task.id) os.environ['FREPPLE_DATABASE'] = database os.environ['PATH'] = settings.FREPPLE_HOME + os.pathsep + os.environ['PATH'] + os.pathsep + settings.FREPPLE_APP if os.path.isfile(os.path.join(settings.FREPPLE_HOME,'libfrepple.so')): os.environ['LD_LIBRARY_PATH'] = settings.FREPPLE_HOME if 'DJANGO_SETTINGS_MODULE' not in os.environ: os.environ['DJANGO_SETTINGS_MODULE'] = 'freppledb.settings' if os.path.exists(os.path.join(settings.FREPPLE_HOME,'python27.zip')): # For the py2exe executable os.environ['PYTHONPATH'] = os.path.join(settings.FREPPLE_HOME,'python27.zip') + os.pathsep + os.path.normpath(settings.FREPPLE_APP) else: # Other executables os.environ['PYTHONPATH'] = os.path.normpath(settings.FREPPLE_APP) ret = os.system('frepple "%s"' % cmd.replace('\\','\\\\')) if ret != 0 and ret != 2: # Return code 0 is a successful run # Return code is 2 is a run cancelled by a user. That's shown in the status field. raise Exception('Failed with exit code %d' % ret) # Task update task.status = 'Done' task.finished = datetime.now() except Exception as e: if task: task.status = 'Failed' task.message = '%s' % e task.finished = datetime.now() raise e finally: if task: task.save(using=database) try: transaction.commit(using=database) except: pass transaction.leave_transaction_management(using=database)
def handle(self, *fixture_labels, **options): using = options.get('database', DEFAULT_DB_ALIAS) connection = connections[using] self.style = no_style() verbosity = int(options.get('verbosity', 1)) show_traceback = options.get('traceback', False) # commit is a stealth option - it isn't really useful as # a command line option, but it can be useful when invoking # loaddata from within another script. # If commit=True, loaddata will use its own transaction; # if commit=False, the data load SQL will become part of # the transaction in place when loaddata was invoked. commit = options.get('commit', True) # Keep a count of the installed objects and fixtures fixture_count = 0 loaded_object_count = 0 fixture_object_count = 0 models = set() humanize = lambda dirname: dirname and "'%s'" % dirname or 'absolute path' # Get a cursor (even though we don't need one yet). This has # the side effect of initializing the test database (if # it isn't already initialized). cursor = connection.cursor() # Start transaction management. All fixtures are installed in a # single transaction to ensure that all references are resolved. if commit: transaction.commit_unless_managed(using=using) transaction.enter_transaction_management(using=using) transaction.managed(True, using=using) class SingleZipReader(zipfile.ZipFile): def __init__(self, *args, **kwargs): zipfile.ZipFile.__init__(self, *args, **kwargs) if settings.DEBUG: assert len( self.namelist() ) == 1, "Zip-compressed fixtures must contain only one file." def read(self): return zipfile.ZipFile.read(self, self.namelist()[0]) compression_types = { None: file, 'gz': gzip.GzipFile, 'zip': SingleZipReader } if has_bz2: compression_types['bz2'] = bz2.BZ2File # from https://code.djangoproject.com/attachment/ticket/3615/defer_constraint_checks.diff connection.begin_defer_constraint_checks() app_module_paths = [] for app in get_apps(): if hasattr(app, '__path__'): # It's a 'models/' subpackage for path in app.__path__: app_module_paths.append(path) else: # It's a models.py module app_module_paths.append(app.__file__) app_fixtures = [ os.path.join(os.path.dirname(path), 'fixtures') for path in app_module_paths ] for fixture_label in fixture_labels: parts = fixture_label.split('.') if len(parts) > 1 and parts[-1] in compression_types: compression_formats = [parts[-1]] parts = parts[:-1] else: compression_formats = compression_types.keys() if len(parts) == 1: fixture_name = parts[0] formats = serializers.get_public_serializer_formats() else: fixture_name, format = '.'.join(parts[:-1]), parts[-1] if format in serializers.get_public_serializer_formats(): formats = [format] else: formats = [] if formats: if verbosity > 1: self.stdout.write("Loading '%s' fixtures...\n" % fixture_name) else: self.stderr.write( self.style.ERROR( "Problem installing fixture '%s': %s is not a known serialization format.\n" % (fixture_name, format))) # from https://code.djangoproject.com/attachment/ticket/3615/defer_constraint_checks.diff connection.end_defer_constraint_checks() if commit: transaction.rollback(using=using) transaction.leave_transaction_management(using=using) return if os.path.isabs(fixture_name): fixture_dirs = [fixture_name] else: fixture_dirs = app_fixtures + list( settings.FIXTURE_DIRS) + [''] for fixture_dir in fixture_dirs: if verbosity > 1: self.stdout.write("Checking %s for fixtures...\n" % humanize(fixture_dir)) label_found = False for combo in product([using, None], formats, compression_formats): database, format, compression_format = combo file_name = '.'.join( p for p in [fixture_name, database, format, compression_format] if p) if verbosity > 1: self.stdout.write("Trying %s for %s fixture '%s'...\n" % \ (humanize(fixture_dir), file_name, fixture_name)) full_path = os.path.join(fixture_dir, file_name) open_method = compression_types[compression_format] try: fixture = open_method(full_path, 'r') if label_found: fixture.close() self.stderr.write( self.style.ERROR( "Multiple fixtures named '%s' in %s. Aborting.\n" % (fixture_name, humanize(fixture_dir)))) # from https://code.djangoproject.com/attachment/ticket/3615/defer_constraint_checks.diff connection.end_defer_constraint_checks() if commit: transaction.rollback(using=using) transaction.leave_transaction_management( using=using) return else: fixture_count += 1 objects_in_fixture = 0 loaded_objects_in_fixture = 0 if verbosity > 0: self.stdout.write("Installing %s fixture '%s' from %s.\n" % \ (format, fixture_name, humanize(fixture_dir))) try: objects = serializers.deserialize(format, fixture, using=using) for obj in objects: objects_in_fixture += 1 if router.allow_syncdb( using, obj.object.__class__): loaded_objects_in_fixture += 1 models.add(obj.object.__class__) obj.save(using=using) loaded_object_count += loaded_objects_in_fixture fixture_object_count += objects_in_fixture label_found = True except (SystemExit, KeyboardInterrupt): raise except Exception: import traceback fixture.close() # from https://code.djangoproject.com/attachment/ticket/3615/defer_constraint_checks.diff connection.end_defer_constraint_checks() if commit: transaction.rollback(using=using) transaction.leave_transaction_management( using=using) if show_traceback: traceback.print_exc() else: self.stderr.write( self.style.ERROR( "Problem installing fixture '%s': %s\n" % (full_path, ''.join( traceback.format_exception( sys.exc_type, sys.exc_value, sys.exc_traceback))))) return fixture.close() # If the fixture we loaded contains 0 objects, assume that an # error was encountered during fixture loading. if objects_in_fixture == 0: self.stderr.write( self.style.ERROR( "No fixture data found for '%s'. (File format may be invalid.)\n" % (fixture_name))) # from https://code.djangoproject.com/attachment/ticket/3615/defer_constraint_checks.diff connection.end_defer_constraint_checks() if commit: transaction.rollback(using=using) transaction.leave_transaction_management( using=using) return except Exception, e: if verbosity > 1: self.stdout.write("No %s fixture '%s' in %s.\n" % \ (format, fixture_name, humanize(fixture_dir)))
def handle(self, *fixture_labels, **options): self.ignore = options.get('ignore') self.using = options.get('database') connection = connections[self.using] if not len(fixture_labels): raise CommandError( "No database fixture specified. Please provide the path of at " "least one fixture in the command line." ) self.verbosity = int(options.get('verbosity')) # commit is a stealth option - it isn't really useful as # a command line option, but it can be useful when invoking # loaddata from within another script. # If commit=True, loaddata will use its own transaction; # if commit=False, the data load SQL will become part of # the transaction in place when loaddata was invoked. commit = options.get('commit', True) # Keep a count of the installed objects and fixtures self.fixture_count = 0 self.loaded_object_count = 0 self.fixture_object_count = 0 self.models = set() # Get a cursor (even though we don't need one yet). This has # the side effect of initializing the test database (if # it isn't already initialized). cursor = connection.cursor() # Start transaction management. All fixtures are installed in a # single transaction to ensure that all references are resolved. if commit: transaction.commit_unless_managed(using=self.using) transaction.enter_transaction_management(using=self.using) transaction.managed(True, using=self.using) class SingleZipReader(zipfile.ZipFile): def __init__(self, *args, **kwargs): zipfile.ZipFile.__init__(self, *args, **kwargs) if settings.DEBUG: assert len(self.namelist()) == 1, "Zip-compressed fixtures must contain only one file." def read(self): return zipfile.ZipFile.read(self, self.namelist()[0]) self.compression_types = { None: open, 'gz': gzip.GzipFile, 'zip': SingleZipReader } if has_bz2: self.compression_types['bz2'] = bz2.BZ2File app_module_paths = [] for app in get_apps(): if hasattr(app, '__path__'): # It's a 'models/' subpackage for path in app.__path__: app_module_paths.append(upath(path)) else: # It's a models.py module app_module_paths.append(upath(app.__file__)) app_fixtures = [os.path.join(os.path.dirname(path), 'fixtures') for path in app_module_paths] try: with connection.constraint_checks_disabled(): for fixture_label in fixture_labels: self.load_label(fixture_label, app_fixtures) # Since we disabled constraint checks, we must manually check for # any invalid keys that might have been added table_names = [model._meta.db_table for model in self.models] try: connection.check_constraints(table_names=table_names) except Exception as e: e.args = ("Problem installing fixtures: %s" % e,) raise except (SystemExit, KeyboardInterrupt): raise except Exception as e: if commit: transaction.rollback(using=self.using) transaction.leave_transaction_management(using=self.using) raise # If we found even one object in a fixture, we need to reset the # database sequences. if self.loaded_object_count > 0: sequence_sql = connection.ops.sequence_reset_sql(no_style(), self.models) if sequence_sql: if self.verbosity >= 2: self.stdout.write("Resetting sequences\n") for line in sequence_sql: cursor.execute(line) if commit: transaction.commit(using=self.using) transaction.leave_transaction_management(using=self.using) if self.verbosity >= 1: if self.fixture_object_count == self.loaded_object_count: self.stdout.write("Installed %d object(s) from %d fixture(s)" % ( self.loaded_object_count, self.fixture_count)) else: self.stdout.write("Installed %d object(s) (of %d) from %d fixture(s)" % ( self.loaded_object_count, self.fixture_object_count, self.fixture_count)) # Close the DB connection. This is required as a workaround for an # edge case in MySQL: if the same connection is used to # create tables, load data, and query, the query can return # incorrect results. See Django #7572, MySQL #37735. if commit: connection.close()