def delete_expired(self, expires): """Delete all expired taskset results.""" self.get_all_expired(expires).update(hidden=True) cursor = self.connection_for_write().cursor() cursor.execute("DELETE FROM %s WHERE hidden=%%s" % ( self.model._meta.db_table, ), (True, )) transaction.commit_unless_managed()
def delete_objects(seen_objs): """ Iterate through a list of seen classes, and remove any instances that are referred to. """ try: ordered_classes = seen_objs.keys() except CyclicDependency: # If there is a cyclic dependency, we cannot in general delete the # objects. However, if an appropriate transaction is set up, or if the # database is lax enough, it will succeed. So for now, we go ahead and # try anyway. ordered_classes = seen_objs.unordered_keys() obj_pairs = {} for cls in ordered_classes: items = seen_objs[cls].items() items.sort() obj_pairs[cls] = items # Pre-notify all instances to be deleted. for pk_val, instance in items: signals.pre_delete.send(sender=cls, instance=instance) pk_list = [pk for pk,instance in items] del_query = sql.DeleteQuery(cls, connection) del_query.delete_batch_related(pk_list) update_query = sql.UpdateQuery(cls, connection) for field, model in cls._meta.get_fields_with_model(): if (field.rel and field.null and field.rel.to in seen_objs and filter(lambda f: f.column == field.column, field.rel.to._meta.fields)): if model: sql.UpdateQuery(model, connection).clear_related(field, pk_list) else: update_query.clear_related(field, pk_list) # Now delete the actual data. for cls in ordered_classes: items = obj_pairs[cls] items.reverse() pk_list = [pk for pk,instance in items] del_query = sql.DeleteQuery(cls, connection) del_query.delete_batch(pk_list) # Last cleanup; set NULLs where there once was a reference to the # object, NULL the primary key of the found objects, and perform # post-notification. for pk_val, instance in items: for field in cls._meta.fields: if field.rel and field.null and field.rel.to in seen_objs: setattr(instance, field.attname, None) signals.post_delete.send(sender=cls, instance=instance) setattr(instance, cls._meta.pk.attname, None) transaction.commit_unless_managed()
def add_root(cls, **kwargs): "Adds a root node to the tree." # do we have a root node already? last_root = cls.get_last_root_node() if last_root and last_root.node_order_by: # there are root nodes and node_order_by has been set # delegate sorted insertion to add_sibling return last_root.add_sibling('sorted-sibling', **kwargs) if last_root: # adding the new root node as the last one newtree_id = last_root.tree_id + 1 else: # adding the first root node newtree_id = 1 # creating the new object newobj = cls(**kwargs) newobj.depth = 1 newobj.tree_id = newtree_id newobj.lft = 1 newobj.rgt = 2 # saving the instance before returning it newobj.save() transaction.commit_unless_managed() return newobj
def migrate_media_field_forwards(model, image_field): cursor = connection.cursor() cursor.execute( "UPDATE %s SET %s = SUBSTR(%s, 7) WHERE %s LIKE 'media%%%%'" % (model._meta.db_table, image_field, image_field, image_field) ) transaction.commit_unless_managed()
def backwards(self, orm): from django.db import connection, transaction cursor = connection.cursor() cursor.execute("UPDATE music_album SET cover = REPLACE(cover, 'dynamic/media', 'media');") transaction.commit_unless_managed() cursor.execute("UPDATE music_song SET tab = REPLACE(tab, 'dynamic/media', 'media');") transaction.commit_unless_managed()
def _queue_requests(): """ Add tableau factory generation requests for each location and factory. We add a record to the TableauFactoryGenerationRequest table for each location and tableau factory that we want to generate. The daemon will then pick up the requests and fire them off, one at a time, to our private API call. """ # Ask the factory generator to assemble a list of all the tableau-creation # requests. requests = factory_generator.calc_generation_requests() # We now have a single large list of requests. These might be too big to # add to the database all at once, so break them into chunks and add them # one chunk at a time. cursor = connections['default'].cursor() for chunk in _splitList(requests, 1000): cmd = "INSERT INTO tableau_editor_tableaufactorygenerationrequest " \ + "(factory_id, loc_code) VALUES (%s, %s)" if len(chunk) == 1: cursor.execute(cmd, chunk[0]) else: cursor.executemany(cmd, chunk) transaction.commit_unless_managed() # Commit changes after each chunk.
def add_root(cls, **kwargs): """ Adds a root node to the tree. :raise PathOverflow: when no more root objects can be added """ # do we have a root node already? last_root = cls.get_last_root_node() if last_root and last_root.node_order_by: # there are root nodes and node_order_by has been set # delegate sorted insertion to add_sibling return last_root.add_sibling('sorted-sibling', **kwargs) if last_root: # adding the new root node as the last one newpath = cls._inc_path(last_root.path) else: # adding the first root node newpath = cls._get_path(None, 1, 1) # creating the new object newobj = cls(**kwargs) newobj.depth = 1 newobj.path = newpath # saving the instance before returning it newobj.save() transaction.commit_unless_managed() return newobj
def handle_noargs(self, **options): """ Command execution. """ self.cursor = connection.cursor() self.introspection = connection.introspection found_missing_fields = False models = translator.get_registered_models(abstract=False) for model in models: db_table = model._meta.db_table model_full_name = '%s.%s' % (model._meta.app_label, model._meta.module_name) opts = translator.get_options_for_model(model) for field_name in opts.local_fields.iterkeys(): missing_langs = list(self.get_missing_languages(field_name, db_table)) if missing_langs: found_missing_fields = True print_missing_langs(missing_langs, field_name, model_full_name) sql_sentences = self.get_sync_sql(field_name, missing_langs, model) execute_sql = ask_for_confirmation(sql_sentences, model_full_name) if execute_sql: print 'Executing SQL...', for sentence in sql_sentences: self.cursor.execute(sentence) print 'Done' else: print 'SQL not executed' transaction.commit_unless_managed() if not found_missing_fields: print 'No new translatable fields detected'
def load_bulk(cls, bulk_data, parent=None, keep_ids=False): "Loads a list/dictionary structure to the tree." # tree, iterative preorder added = [] if parent: parent_id = parent.id else: parent_id = None # stack of nodes to analize stack = [(parent_id, node) for node in bulk_data[::-1]] while stack: parent_id, node_struct = stack.pop() # shallow copy of the data strucure so it doesn't persist... node_data = node_struct['data'].copy() if keep_ids: node_data['id'] = node_struct['id'] if parent_id: parent = cls.objects.get(pk=parent_id) node_obj = parent.add_child(**node_data) else: node_obj = cls.add_root(**node_data) added.append(node_obj.id) if 'children' in node_struct: # extending the stack with the current node as the parent of # the new nodes stack.extend([(node_obj.id, node) \ for node in node_struct['children'][::-1]]) transaction.commit_unless_managed() return added
def handle_label(self, tablename, **options): alias = options.get('database', DEFAULT_DB_ALIAS) connection = connections[alias] fields = ( # "key" is a reserved word in MySQL, so use "cache_key" instead. models.CharField(name='cache_key', max_length=255, unique=True, primary_key=True), models.TextField(name='value'), models.DateTimeField(name='expires', db_index=True), ) table_output = [] index_output = [] qn = connection.ops.quote_name for f in fields: field_output = [qn(f.name), f.db_type(connection=connection)] field_output.append("%sNULL" % (not f.null and "NOT " or "")) if f.primary_key: field_output.append("PRIMARY KEY") elif f.unique: field_output.append("UNIQUE") if f.db_index: unique = f.unique and "UNIQUE " or "" index_output.append("CREATE %sINDEX %s ON %s (%s);" % \ (unique, qn('%s_%s' % (tablename, f.name)), qn(tablename), qn(f.name))) table_output.append(" ".join(field_output)) full_statement = ["CREATE TABLE %s (" % qn(tablename)] for i, line in enumerate(table_output): full_statement.append(' %s%s' % (line, i < len(table_output) - 1 and ',' or '')) full_statement.append(');') curs = connection.cursor() curs.execute("\n".join(full_statement)) for statement in index_output: curs.execute(statement) transaction.commit_unless_managed(using=alias)
def install(self): try: with transaction.atomic(): self.install_atomic() except AttributeError: self.install_atomic() transaction.commit_unless_managed(using=self.using)
def fill_table(sheet, columns, name): cursor = connection.cursor() column_sql = """date, player_id, player_surname, player_forename, team, team_id, opposition, opposition_id, venue, """ column_sql += ', '.join(columns) for row in range(sheet.nrows): if row == 0: continue row_values = [ ] count = 0 for value in sheet.row_values(row): if count in varchar_column_indices: row_values.append("%s" % ('"' + value.encode('utf-8') + '"')) else: row_values.append(str(int(value)).encode('utf-8')) count += 1 values_sql = ','.join(row_values) sql = 'INSERT INTO %s (%s) VALUES (%s);' % (str(name), str(column_sql), str(values_sql)) # print sql cursor.execute(sql) transaction.commit_unless_managed()
def initialize_source_progress(lt_rlz, hzrd_src): """ Create ``source_progress`` models for given logic tree realization and set total sources of realization. :param lt_rlz: :class:`openquake.engine.db.models.LtRealization` object to initialize source progress for. :param hztd_src: :class:`openquake.engine.db.models.Input` object that needed parsed sources are referencing. """ cursor = connections["reslt_writer"].cursor() src_progress_tbl = models.SourceProgress._meta.db_table parsed_src_tbl = models.ParsedSource._meta.db_table lt_rlz_tbl = models.LtRealization._meta.db_table cursor.execute( """ INSERT INTO "%s" (lt_realization_id, parsed_source_id, is_complete) SELECT %%s, id, FALSE FROM "%s" WHERE input_id = %%s ORDER BY id """ % (src_progress_tbl, parsed_src_tbl), [lt_rlz.id, hzrd_src.id], ) cursor.execute( """ UPDATE "%s" SET total_items = ( SELECT count(1) FROM "%s" WHERE lt_realization_id = %%s )""" % (lt_rlz_tbl, src_progress_tbl), [lt_rlz.id], ) transaction.commit_unless_managed()
def wrapped(*args, **kw): was_exception = False try: return fun(*args, **kw) except: was_exception = True # Log the exception so we can actually see it in procuction. # When celery is upgraded to 2.2, this can be done more # gracefully with a signal. # See: http://groups.google.com/group/celery-users/ # browse_thread/thread/95bdffe5a0057ac0?pli=1 exc_info = sys.exc_info() log.error(u'Celery TASK exception: %s: %s' % (exc_info[1].__class__.__name__, exc_info[1]), exc_info=exc_info) raise finally: try: for db in connections: transaction.commit_unless_managed(using=db) except: if was_exception: # We want to see the original exception so let's # just log the one after that. log.exception( 'While trying to recover from an exception') else: raise
def _clear_items(self, source_col_name): # source_col_name: the PK colname in join_table for the source object cursor = connection.cursor() cursor.execute("DELETE FROM %s WHERE %s = %%s" % \ (self.join_table, source_col_name), [self._pk_val]) transaction.commit_unless_managed()
def handle_noargs(self, **options): cursor = connection.cursor() sql_files = glob.glob(os.path.join(settings.MIGRATIONS_ROOT, '*.sql')) py_files = glob.glob(os.path.join(settings.MIGRATIONS_ROOT, '*.py')) files = sorted(sql_files + py_files) alter_db = False for fn in files: done = '%s.done' % fn if not os.path.exists(done): if fn.endswith('sql'): f = open(fn, 'r') sql = f.read() f.close() print "Executing: %s" % os.path.split(fn)[1] cursor.execute(sql) transaction.commit_unless_managed() alter_db = True else: name, _ = os.path.splitext(os.path.split(fn)[1]) if name.startswith("__"): continue __import__('migrations', globals(), locals(), [name]) alter_db = sys.modules['migrations.%s' % name].main() open(done, 'w').close() if not alter_db: print "Up to date, nothing to do." else: print "Success!"
def log_answer(answer): pin_this_thread() creator = answer.creator created = answer.created question = answer.question users = [a.creator for a in question.answers.select_related('creator').exclude( creator=creator)] if question.creator != creator: users += [question.creator] users = set(users) # Remove duplicates. if users: action = Action.objects.create( creator=creator, created=created, url=answer.get_absolute_url(), content_object=answer, formatter_cls='questions.formatters.AnswerFormatter') action.users.add(*users) transaction.commit_unless_managed() # Record karma actions AnswerAction(answer.creator, answer.created.date()).save() if answer == answer.question.answers.order_by('created')[0]: FirstAnswerAction(answer.creator, answer.created.date()).save() unpin_this_thread()
def weekly_downloads(): """ Update 7-day add-on download counts. """ cursor = connection.cursor() cursor.execute(""" SELECT addon_id, SUM(count) AS weekly_count FROM download_counts WHERE `date` >= DATE_SUB(CURDATE(), INTERVAL 7 DAY) GROUP BY addon_id ORDER BY addon_id""") counts = cursor.fetchall() addon_ids = [r[0] for r in counts] if not addon_ids: return cursor.execute(""" SELECT id, 0 FROM addons WHERE id NOT IN %s""", (addon_ids,)) counts += cursor.fetchall() cursor.execute(""" CREATE TEMPORARY TABLE tmp (addon_id INT PRIMARY KEY, count INT)""") cursor.execute('INSERT INTO tmp VALUES %s' % ','.join(['(%s,%s)'] * len(counts)), list(itertools.chain(*counts))) cursor.execute(""" UPDATE addons INNER JOIN tmp ON addons.id = tmp.addon_id SET weeklydownloads = tmp.count""") transaction.commit_unless_managed()
def _remove_items(self, source_col_name, target_col_name, *objs): # source_col_name: the PK colname in join_table for the source object # target_col_name: the PK colname in join_table for the target object # *objs - objects to remove # If there aren't any objects, there is nothing to do. if objs: # Check that all the objects are of the right type old_ids = set() for obj in objs: if isinstance(obj, self.model): old_ids.add(obj._get_pk_val()) else: old_ids.add(obj) # Remove the specified objects from the join table cursor = connection.cursor() cursor.execute("DELETE FROM %s WHERE %s = %%s AND %s IN (%s)" % \ (self.join_table, source_col_name, target_col_name, ",".join(['%s'] * len(old_ids))), [self._pk_val] + list(old_ids)) if self.reverse: sender = self.model else: sender = self.instance.__class__ signals.m2m_changed.send(sender=sender, action="remove", instance=self.instance, model=self.model, reverse=self.reverse, field_name=self.field_name, objects=list(objs)) transaction.commit_unless_managed()
def set_no_similarity(request, osm, osm_from, product_id_to, product_id_from): response = {} if request.method == 'POST': nosim = list(NoProductSimilarity.objects.raw("SELECT * FROM matcher_noproductsimilarity WHERE "+osm+"_product_id = %s and "+osm_from+"_product_id = %s", (product_id_to, product_id_from))) if len(nosim): # Relation of no similarity already exists, do not do anything response['status'] = 200 response['msg'] = 'Done' else: # The similarity does not exist, create it # sql_insert = # INSERT QUERY cursor = connection.cursor() cursor.execute("INSERT INTO matcher_noproductsimilarity ("+osm+"_product_id, "+osm_from+"_product_id) VALUES ( %s , %s )", [product_id_to, product_id_from]) transaction.commit_unless_managed() response['status'] = 200 response['msg'] = 'Added to database' elif request.method == 'DELETE': nosim = list(NoProductSimilarity.objects.raw("SELECT * FROM matcher_noproductsimilarity WHERE "+osm+"_product_id = %s and "+osm_from+"_product_id = %s", (product_id_to, product_id_from))) if len(nosim)>0: nosim = nosim[0] nosim.delete() response['status'] = 200 response['msg'] = 'Deleted' else: response['status'] = 200 response['msg'] = 'Nothing to do' else: response['status'] = 403 response['msg'] = 'Method not handled' return HttpResponse(json.dumps(response))
def notification_remove_cc(request, case_ids, cc_list): ''' Description: Remove email addresses from the notification CC list of specific TestCases Params: $case_ids - Integer/Array: one or more TestCase IDs $cc_list - Array: contians the email addresses that will be removed from each TestCase indicated by case_ids. Returns: JSON. When succeed, status is 0, and message maybe empty or anything else that depends on the implementation. If something wrong, status will be 1 and message will be a short description to the error. ''' try: validate_cc_list(cc_list) except (TypeError, ValidationError): raise try: tc_ids = pre_process_ids(case_ids) cursor = connection.writer_cursor ids_values = ",".join(itertools.repeat('%s', len(tc_ids))) email_values = ",".join(itertools.repeat('%s', len(cc_list))) sql = TC_REMOVE_CC % (ids_values, email_values) tc_ids.extend(cc_list) cursor.execute(sql, tc_ids) transaction.commit_unless_managed() except (TypeError, ValueError, Exception): raise
def migrate_media_field_backwards(model, image_field): cursor = connection.cursor() cursor.execute( "UPDATE %s SET %s ='media/' || %s WHERE %s != ''" % (model._meta.db_table, image_field, image_field, image_field) ) transaction.commit_unless_managed()
def handle(self, **options): '''The main entry point for the Django management command.''' import_start = datetime.datetime.now() # Start transaction management. transaction.commit_unless_managed() transaction.enter_transaction_management() transaction.managed(True) try: self._promote_devel() except: self._rollback_db() raise # Finalize the transaction and close the db connection. transaction.commit() transaction.leave_transaction_management() connection.close() import_end = datetime.datetime.now() # Print a short summary of what we did. td = import_end - import_start print '\nProcessing complete in %s days, %s.%s seconds.' % ( td.days, td.seconds, td.microseconds) print ' TraitData objects promoted: %s' % ( PublicTraitData.objects.all().count(),)
def get_descendants_group_count(cls, parent=None): """ Helper for a very common case: get a group of siblings and the number of *descendants* in every sibling. """ #~ # disclaimer: this is the FOURTH implementation I wrote for this # function. I really tried to make it return a queryset, but doing so # with a *single* query isn't trivial with Django's ORM. # ok, I DID manage to make Django's ORM return a queryset here, # defining two querysets, passing one subquery in the tables parameters # of .extra() of the second queryset, using the undocumented order_by # feature, and using a HORRIBLE hack to avoid django quoting the # subquery as a table, BUT (and there is always a but) the hack didn't # survive turning the QuerySet into a ValuesQuerySet, so I just used # good old SQL. # NOTE: in case there is interest, the hack to avoid django quoting the # subquery as a table, was adding the subquery to the alias cache of # the queryset's query object: # # qset.query.quote_cache[subquery] = subquery # # If there is a better way to do this in an UNMODIFIED django 1.0, let # me know. #~ if parent: depth = parent.depth + 1 params = cls._get_children_path_interval(parent.path) extrand = 'AND path BETWEEN %s AND %s' else: depth = 1 params = [] extrand = '' sql = 'SELECT * FROM %(table)s AS t1 INNER JOIN '\ ' (SELECT '\ ' SUBSTR(path, 1, %(subpathlen)s) AS subpath, '\ ' COUNT(1)-1 AS count '\ ' FROM %(table)s '\ ' WHERE depth >= %(depth)s %(extrand)s'\ ' GROUP BY subpath) AS t2 '\ ' ON t1.path=t2.subpath '\ ' ORDER BY t1.path' % { 'table': connection.ops.quote_name(cls.tb_table_name), 'subpathlen': depth * cls.steplen, 'depth': depth, 'extrand': extrand} cursor = cls._get_database_cursor('write') cursor.execute(sql, params) ret = [] field_names = [field[0] for field in cursor.description] for node_data in cursor.fetchall(): node = cls(**dict(zip(field_names, node_data[:-2]))) node.descendants_count = node_data[-1] ret.append(node) transaction.commit_unless_managed() return ret
def collection_subscribers(): """ Collection weekly and monthly subscriber counts. """ log.debug('Starting collection subscriber update...') cursor = connection.cursor() cursor.execute(""" UPDATE collections SET weekly_subscribers = 0, monthly_subscribers = 0 """) cursor.execute(""" UPDATE collections AS c INNER JOIN ( SELECT COUNT(collection_id) AS count, collection_id FROM collection_subscriptions WHERE created >= DATE_SUB(CURDATE(), INTERVAL 7 DAY) GROUP BY collection_id ) AS weekly ON (c.id = weekly.collection_id) INNER JOIN ( SELECT COUNT(collection_id) AS count, collection_id FROM collection_subscriptions WHERE created >= DATE_SUB(CURDATE(), INTERVAL 31 DAY) GROUP BY collection_id ) AS monthly ON (c.id = monthly.collection_id) SET c.weekly_subscribers = weekly.count, c.monthly_subscribers = monthly.count """) transaction.commit_unless_managed()
def update(self, **kwargs): """ Updates all elements in the current QuerySet, setting all the given fields to the appropriate values. """ assert self.query.can_filter(), \ "Cannot update a query once a slice has been taken." query = self.query.clone(sql.UpdateQuery) query.add_update_values(kwargs) if not transaction.is_managed(): transaction.enter_transaction_management() forced_managed = True else: forced_managed = False try: rows = query.execute_sql(None) if forced_managed: transaction.commit() else: transaction.commit_unless_managed() finally: if forced_managed: transaction.leave_transaction_management() self._result_cache = None return rows
def clean_sessions(): """ Queue deleting expired session items without breaking poor MySQL """ now = timezone.now() logger = clean_sessions.get_logger() chunk_size = constance.config.SESSION_CLEANUP_CHUNK_SIZE if memcache.add(LOCK_ID, now.strftime('%c'), LOCK_EXPIRE): total_count = get_expired_sessions(now).count() delete_count = 0 logger.info('Deleting the %s of %s oldest expired sessions' % (chunk_size, total_count)) try: cursor = connection.cursor() delete_count = cursor.execute(""" DELETE FROM django_session WHERE expire_date < NOW() ORDER BY expire_date ASC LIMIT %s; """, [chunk_size]) transaction.commit_unless_managed() finally: logger.info('Deleted %s expired sessions' % delete_count) memcache.delete(LOCK_ID) expired_sessions = get_expired_sessions(now) if expired_sessions.exists(): clean_sessions.apply_async() else: logger.error('The clean_sessions task is already running since %s' % memcache.get(LOCK_ID))
def move_node(self, node, target, position='last-child'): """ Moves ``node`` relative to a given ``target`` node as specified by ``position`` (when appropriate), by examining both nodes and calling the appropriate method to perform the move. A ``target`` of ``None`` indicates that ``node`` should be turned into a root node. Valid values for ``position`` are ``'first-child'``, ``'last-child'``, ``'left'`` or ``'right'``. ``node`` will be modified to reflect its new tree state in the database. This method explicitly checks for ``node`` being made a sibling of a root node, as this is a special case due to our use of tree ids to order root nodes. """ if target is None: if node.is_child_node(): self._make_child_root_node(node) elif target.is_root_node() and position in ['left', 'right']: self._make_sibling_of_root_node(node, target, position) else: if node.is_root_node(): self._move_root_node(node, target, position) else: self._move_child_node(node, target, position) transaction.commit_unless_managed()
def _base_set(self, mode, key, value, timeout=None): if timeout is None: timeout = self.default_timeout cursor = connection.cursor() cursor.execute("SELECT COUNT(*) FROM %s" % self._table) num = cursor.fetchone()[0] now = datetime.now().replace(microsecond=0) exp = datetime.fromtimestamp(time.time() + timeout).replace(microsecond=0) if num > self._max_entries: self._cull(cursor, now) encoded = base64.encodestring(pickle.dumps(value, 2)).strip() cursor.execute("SELECT cache_key, expires FROM %s WHERE cache_key = %%s" % self._table, [key]) try: result = cursor.fetchone() if result and (mode == 'set' or (mode == 'add' and result[1] < now)): cursor.execute("UPDATE %s SET value = %%s, expires = %%s WHERE cache_key = %%s" % self._table, [encoded, str(exp), key]) else: cursor.execute("INSERT INTO %s (cache_key, value, expires) VALUES (%%s, %%s, %%s)" % self._table, [key, encoded, str(exp)]) except DatabaseError: # To be threadsafe, updates/inserts are allowed to fail silently return False else: transaction.commit_unless_managed() return True
def _add_items(self, source_col_name, target_col_name, *objs): # join_table: name of the m2m link table # source_col_name: the PK colname in join_table for the source object # target_col_name: the PK colname in join_table for the target object # *objs - objects to add. Either object instances, or primary keys of object instances. # If there aren't any objects, there is nothing to do. if objs: from django.db.models.base import Model # Check that all the objects are of the right type new_ids = set() for obj in objs: if isinstance(obj, self.model): new_ids.add(obj._get_pk_val()) elif isinstance(obj, Model): raise TypeError, "'%s' instance expected" % self.model._meta.object_name else: new_ids.add(obj) # Add the newly created or already existing objects to the join table. # First find out which items are already added, to avoid adding them twice cursor = connection.cursor() cursor.execute("SELECT %s FROM %s WHERE %s = %%s AND %s IN (%s)" % \ (target_col_name, self.join_table, source_col_name, target_col_name, ",".join(['%s'] * len(new_ids))), [self._pk_val] + list(new_ids)) existing_ids = set([row[0] for row in cursor.fetchall()]) # Add the ones that aren't there already for obj_id in (new_ids - existing_ids): cursor.execute("INSERT INTO %s (%s, %s) VALUES (%%s, %%s)" % \ (self.join_table, source_col_name, target_col_name), [self._pk_val, obj_id]) transaction.commit_unless_managed()
def handle_app(self, app, **options): using = options.get('database', DEFAULT_DB_ALIAS) connection = connections[using] app_name = app.__name__.split('.')[-2] self.style = no_style() sql_list = sql_reset(app, self.style, connection) if options.get('interactive'): confirm = raw_input(""" You have requested a database reset. This will IRREVERSIBLY DESTROY any data for the "%s" application in the database "%s". Are you sure you want to do this? Type 'yes' to continue, or 'no' to cancel: """ % (app_name, connection.settings_dict['NAME'])) else: confirm = 'yes' if confirm == 'yes': try: cursor = connection.cursor() for sql in sql_list: cursor.execute(sql) except Exception, e: transaction.rollback_unless_managed() raise CommandError( """Error: %s couldn't be reset. Possible reasons: * The database isn't running or isn't configured correctly. * At least one of the database tables doesn't exist. * The SQL was invalid. Hint: Look at the output of 'django-admin.py sqlreset %s'. That's the SQL this command wasn't able to run. The full error: %s""" % (app_name, app_name, e)) transaction.commit_unless_managed()
def set_default_screenshots(): cur = connection.cursor() cur2 = connection.cursor() # unset default_screenshot for productions that don't have any processed screenshots # (if things are working properly then we would expect this to be redundant. But no harm in being thorough :-) ) cur.execute(''' UPDATE demoscene_production SET default_screenshot_id = NULL WHERE id NOT IN (SELECT DISTINCT production_id FROM demoscene_screenshot WHERE thumbnail_url <> '') ''') # get all processed screenshots (= ones that have thumbnails), group by production_id cur.execute( "SELECT id, production_id FROM demoscene_screenshot WHERE thumbnail_url <> '' ORDER BY production_id" ) for production_id, screenshots in groupby(cur, lambda row: row[1]): # select a random screenshot from the group and set it as default_screenshot for that prod screenshot_id = random.choice([row[0] for row in screenshots]) cur2.execute( ''' UPDATE demoscene_production SET default_screenshot_id = %s WHERE id = %s ''', [screenshot_id, production_id]) transaction.commit_unless_managed()
def weekly_downloads(): """ Update 7-day add-on download counts. """ raise_if_reindex_in_progress() cursor = connection.cursor() cursor.execute(""" SELECT addon_id, SUM(count) AS weekly_count FROM download_counts WHERE `date` >= DATE_SUB(CURDATE(), INTERVAL 7 DAY) GROUP BY addon_id ORDER BY addon_id""") counts = cursor.fetchall() addon_ids = [r[0] for r in counts] if not addon_ids: return cursor.execute( """ SELECT id, 0 FROM addons WHERE id NOT IN %s""", (addon_ids, )) counts += cursor.fetchall() cursor.execute(""" CREATE TEMPORARY TABLE tmp_wd (addon_id INT PRIMARY KEY, count INT)""") cursor.execute( 'INSERT INTO tmp_wd VALUES %s' % ','.join(['(%s,%s)'] * len(counts)), list(itertools.chain(*counts))) cursor.execute(""" UPDATE addons INNER JOIN tmp_wd ON addons.id = tmp_wd.addon_id SET weeklydownloads = tmp_wd.count""") cursor.execute("DROP TABLE IF EXISTS tmp_wd") transaction.commit_unless_managed()
def load_bulk(cls, bulk_data, parent=None, keep_ids=False): """Loads a list/dictionary structure to the tree.""" cls = get_result_class(cls) # tree, iterative preorder added = [] if parent: parent_id = parent.pk else: parent_id = None # stack of nodes to analize stack = [(parent_id, node) for node in bulk_data[::-1]] foreign_keys = cls.get_foreign_keys() while stack: parent_id, node_struct = stack.pop() # shallow copy of the data strucure so it doesn't persist... node_data = node_struct['data'].copy() cls._process_foreign_keys(foreign_keys, node_data) if keep_ids: node_data['id'] = node_struct['id'] if parent_id: parent = cls.objects.get(pk=parent_id) node_obj = parent.add_child(**node_data) else: node_obj = cls.add_root(**node_data) added.append(node_obj.pk) if 'children' in node_struct: # extending the stack with the current node as the parent of # the new nodes stack.extend([ (node_obj.pk, node) for node in node_struct['children'][::-1] ]) transaction.commit_unless_managed() return added
def sync_db(self): ''' This function uses the same code that's used in syncdb to dynamically execute DDL sql on-the-fly. Copied from: /usr/local/lib/python2.6/dist-packages/django/core/management/commands/syncdb.py ''' from django.core.management.color import no_style from django.db import connection, transaction cursor = connection.cursor() tables = connection.introspection.table_names() seen_models = connection.introspection.installed_models(tables) created_models = set() pending_references = {} sql, references = connection.creation.sql_create_model( self.model_class, no_style(), seen_models) seen_models.add(self.model_class) created_models.add(self.model_class) for refto, refs in references.items(): pending_references.setdefault(refto, []).extend(refs) if refto in seen_models: sql.extend( connection.creation.sql_for_pending_references( refto, no_style(), pending_references)) sql.extend( connection.creation.sql_for_pending_references( self.model_class, no_style(), pending_references)) #append point geometry by calling the PostGIS function: sql.append( "select AddGeometryColumn('public','%s','point',4326,'POINT',2)" % self.form.table_name) for statement in sql: cursor.execute(statement) transaction.commit_unless_managed()
def move_node(self, node, target, position='last-child'): """ Moves ``node`` relative to a given ``target`` node as specified by ``position`` (when appropriate), by examining both nodes and calling the appropriate method to perform the move. A ``target`` of ``None`` indicates that ``node`` should be turned into a root node. Valid values for ``position`` are ``'first-child'``, ``'last-child'``, ``'left'`` or ``'right'``. ``node`` will be modified to reflect its new tree state in the database. This method explicitly checks for ``node`` being made a sibling of a root node, as this is a special case due to our use of tree ids to order root nodes. """ if self._base_manager: return self._base_manager.move_node(node, target, position=position) if target is None: if node.is_child_node(): self._make_child_root_node(node) elif target.is_root_node() and position in ['left', 'right']: self._make_sibling_of_root_node(node, target, position) else: if node.is_root_node(): self._move_root_node(node, target, position) else: self._move_child_node(node, target, position) transaction.commit_unless_managed()
def add_root(cls, **kwargs): """Adds a root node to the tree.""" # do we have a root node already? last_root = cls.get_last_root_node() if last_root and last_root.node_order_by: # there are root nodes and node_order_by has been set # delegate sorted insertion to add_sibling return last_root.add_sibling('sorted-sibling', **kwargs) if last_root: # adding the new root node as the last one newtree_id = last_root.tree_id + 1 else: # adding the first root node newtree_id = 1 if len(kwargs) == 1 and 'instance' in kwargs: # adding the passed (unsaved) instance to the tree newobj = kwargs['instance'] if newobj.pk: raise NodeAlreadySaved("Attempted to add a tree node that is "\ "already in the database") else: # creating the new object newobj = get_result_class(cls)(**kwargs) newobj.depth = 1 newobj.tree_id = newtree_id newobj.lft = 1 newobj.rgt = 2 # saving the instance before returning it newobj.save() transaction.commit_unless_managed() return newobj
def add_child(self, **kwargs): "Adds a child to the node." if not self.is_leaf(): # there are child nodes, delegate insertion to add_sibling if self.node_order_by: pos = 'sorted-sibling' else: pos = 'last-sibling' last_child = self.get_last_child() last_child._cached_parent_obj = self return last_child.add_sibling(pos, **kwargs) # we're adding the first child of this node sql, params = self.__class__._move_right(self.tree_id, self.rgt, False, 2) # creating a new object newobj = self.__class__(**kwargs) newobj.tree_id = self.tree_id newobj.depth = self.depth + 1 newobj.lft = self.lft + 1 newobj.rgt = self.lft + 2 # this is just to update the cache self.rgt = self.rgt + 2 newobj._cached_parent_obj = self cursor = connection.cursor() cursor.execute(sql, params) # saving the instance before returning it newobj.save() transaction.commit_unless_managed() return newobj
def bulk_insert_ignore(self,fields,value_list,print_sql=False): """主要解决mysql重复数据过滤的问题 加入了 ignore 关键词""" db_table = self.model._meta.db_table values_sql="(%s)" %(','.join([" %s " for _ in range(len(fields))])) base_sql = "INSERT IGNORE INTO %s (%s) VALUES " % (db_table, ",".join(fields)) sql = """%s %s""" % (base_sql, values_sql) from django.db import connection,transaction cursor = connection.cursor() try: f=cursor.executemany(sql, value_list) k=transaction.commit_unless_managed() import pdb;pdb.set_trace() return True except Exception as e: print e return False
class DatabaseCache(BaseDatabaseCache): # This class uses cursors provided by the database connection. This means # it reads expiration values as aware or naive datetimes depending on the # value of USE_TZ. They must be compared to aware or naive representations # of "now" respectively. # But it bypasses the ORM for write operations. As a consequence, aware # datetimes aren't made naive for databases that don't support time zones. 市区 # We work around this problem by always using naive datetimes when writing # expiration values 过期值, in UTC when USE_TZ = True and in local time otherwise. def get(self, key, default=None, version=None): key = self.make_key(key, version=version) self.validate_key(key) self.cache_model_class 不懂 db = router.db_for_read(self.cache_model_class) table = connections[db].ops.quote_name(self._table) cursor = connections[db].cursor() 游标 cursor.execute("SELECT cache_key, value, expires FROM %s " "WHERE cache_key = %%s" % table, [key]) table row = cursor.fetchone() 取一个 if row is None: 空集 return default now = timezone.now() row[2] 是时间 if row[2] < now: 事件比较新 db = router.db_for_write(self.cache_model_class) cursor = connections[db].cursor() cursor.execute("DELETE FROM %s " "WHERE cache_key = %%s" % table, [key]) transaction.commit_unless_managed(using=db) 事务 return default
class Migration(SchemaMigration): def forwards(self, orm): cursor = connection.cursor() for model in ('urltoken', 'offsiteinvite'): try: cursor.execute("SELECT * FROM registration_%s LIMIT 1" % model) db.rename_table('registration_%s' % model, 'jmbo_registration_%s' % model) if orm['contenttypes.ContentType'].objects.filter( app_label='registration', model=model).exists(): orm['contenttypes.ContentType'].objects.filter( app_label='jmbo_registration', model=model).delete() orm['contenttypes.ContentType'].objects.filter( app_label='registration', model=model).update(app_label='jmbo_registration') except DatabaseError, e: if 'does not exist' not in e.message: raise e transaction.commit_unless_managed()
class Command(BaseCommand): """ Example: python manage.py drop corporate_memberships memberships """ def run_sql(self, sql, **options): """ Parse through sql statements. Execute each sql statement. Commit in one transaction. """ # sql statement regex statements = re.compile(r";[ \t]*$", re.M) sql_list = [] # database information using = options.get('database', DEFAULT_DB_ALIAS) connection = connections[using] for statement in statements.split(sql.decode(settings.FILE_CHARSET)): # Remove any comments from the file statement = re.sub(ur"--.*([\n\Z]|$)", "", statement) if statement.strip(): sql_list.append(statement + u";") try: cursor = connection.cursor() cursor.execute('SET FOREIGN_KEY_CHECKS = 0;') for sql in sql_list: cursor.execute(sql) cursor.execute('SET FOREIGN_KEY_CHECKS = 1;') except Exception, e: transaction.rollback_unless_managed() raise CommandError(""" Error running SQL. Possible reasons: * The database isn't running or isn't configured correctly. * At least one of the database tables doesn't exist. * The SQL was invalid. The full error: %s""" % (e)) transaction.commit_unless_managed() return sql_list
def delete_mp(campaign_id): ad = Ad.objects.get(DE_campaign_id=campaign_id) from django.db import connection, transaction cursor = connection.cursor() cursor.execute('delete from media_planning_de_clickdata where ad_id=%s', [ad.id]) transaction.commit_unless_managed() cursor.execute('delete from media_planning_flight where ad_id=%s', [ad.id]) transaction.commit_unless_managed() cursor.execute('delete from ad_adrelatedstaff where ad_id=%s', [ad.id]) transaction.commit_unless_managed() ad.delete() return True
def actualizar_secuencias(): # select setval('main_familia_id_seq', (select max(id)+1 from main_familia), false) cursor = connection.cursor() cursor.execute( "select setval('main_centrofamiliar_id_seq', (select max(id)+1 from main_centrofamiliar), false)" ) transaction.commit_unless_managed() cursor.execute( "select setval('main_familia_id_seq', (select max(id)+1 from main_familia), false)" ) transaction.commit_unless_managed() cursor.execute( "select setval('main_persona_id_seq', (select max(id)+1 from main_persona), false)" ) transaction.commit_unless_managed()
def clear_trashinfo(): from django.db import connection, transaction cursor = connection.cursor() cursor.execute( 'delete from media_planning_de_clickdata where ad_id not in (select id from ad_ad)' ) transaction.commit_unless_managed() cursor.execute( 'delete from media_planning_flight where ad_id not in (select id from ad_ad)' ) transaction.commit_unless_managed() cursor.execute( 'delete from ad_adrelatedstaff where ad_id not in (select id from ad_ad)' ) transaction.commit_unless_managed() return True
def merge_channel(media_id): # For same media, merge the channels owning the same name # Remove spaces in channels' name # Find same name channels, store them into dict. {channel_name:[channel_id_1, channel_id_2, ...]} try: media = Media.objects.get(id=media_id) except ObjectDoesNotExist: print 'The media dose not exist!' return False channel_name_dict = {} for channel in media.related_channels.all(): channel.c_name = channel.c_name.replace(' ', '') channel.save() if channel_name_dict.has_key(channel.c_name): channel_name_dict[channel.c_name].append(channel.id) else: channel_name_dict[channel.c_name] = [channel.id] # Then, update related tables from django.db import connection, transaction cursor = connection.cursor() for val in channel_name_dict.values(): if len(val) > 1: val.sort() for i in val[1:]: cursor.execute( 'update media_mediaadinfo set channel_id=%s where channel_id=%s', [val[0], i]) transaction.commit_unless_managed() cursor.execute( 'update media_planning_flight set channel_id=%s where channel_id=%s', [val[0], i]) transaction.commit_unless_managed() cursor.execute( 'update media_planning_de_clickdata set channel_id=%s where channel_id=%s', [val[0], i]) transaction.commit_unless_managed() channel = Channel.objects.get(id=i) channel.delete() # At last, call merge_flight channels needed for channel in media.related_channels.all(): merge_flight(channel)
def new_topic(poll): subject = "Poll: %s" % poll.title descr = poll.description game = poll.game now = int(mktime(gmtime())) cursor = connections['fluxbb'].cursor() #cursor.execute("SELECT id FROM users WHERE username = '******'", ['longturn']) cursor.execute("SELECT id FROM users WHERE username = '******'") poster_id = cursor.fetchone()[0] cursor.execute("SELECT id FROM forums WHERE forum_name = '%s'", [AsIs(game)]) forum_id = cursor.fetchone()[0] cmd = "INSERT INTO topics (poster, subject, forum_id, last_poster, posted, last_post) VALUES ('%s', '%s', %d, '%s', %d, %d)" cursor.execute( cmd, ['longturn', AsIs(subject), forum_id, 'longturn', now, now]) transaction.commit_unless_managed(using='fluxbb') cursor.execute( "SELECT max(id) FROM topics WHERE poster = '%s' AND subject = '%s'", ['longturn', AsIs(subject)]) topic_id = cursor.fetchone()[0] cmd = "INSERT INTO posts (poster, poster_id, topic_id, posted, message) VALUES ('%s', %d, %d, %d, '%s')" cursor.execute(cmd, ['longturn', poster_id, topic_id, now, AsIs(descr)]) transaction.commit_unless_managed(using='fluxbb') cmd = "UPDATE forums SET num_topics = num_topics + 1, num_posts = num_posts + 1, last_post = %d, last_poster = '%s' WHERE forum_name = '%s'" cursor.execute(cmd, [now, 'longturn', AsIs(game)]) transaction.commit_unless_managed(using='fluxbb') return topic_id
def fix_tree(cls, destructive=False): """ Solves some problems that can appear when transactions are not used and a piece of code breaks, leaving the tree in an inconsistent state. The problems this method solves are: 1. Nodes with an incorrect ``depth`` or ``numchild`` values due to incorrect code and lack of database transactions. 2. "Holes" in the tree. This is normal if you move/delete nodes a lot. Holes in a tree don't affect performance, 3. Incorrect ordering of nodes when ``node_order_by`` is enabled. Ordering is enforced on *node insertion*, so if an attribute in ``node_order_by`` is modified after the node is inserted, the tree ordering will be inconsistent. :param destructive: A boolean value. If True, a more agressive fix_tree method will be attemped. If False (the default), it will use a safe (and fast!) fix approach, but it will only solve the ``depth`` and ``numchild`` nodes, it won't fix the tree holes or broken path ordering. .. warning:: Currently what the ``destructive`` method does is: 1. Backup the tree with :meth:`dump_data` 2. Remove all nodes in the tree. 3. Restore the tree with :meth:`load_data` So, even when the primary keys of your nodes will be preserved, this method isn't foreign-key friendly. That needs complex in-place tree reordering, not available at the moment (hint: patches are welcome). """ if destructive: dump = cls.dump_bulk(None, True) cls.objects.all().delete() cls.load_bulk(dump, None, True) else: cursor = cls._get_database_cursor('write') # fix the depth field # we need the WHERE to speed up postgres sql = "UPDATE %s "\ "SET depth=LENGTH(path)/%%s "\ "WHERE depth!=LENGTH(path)/%%s" % ( connection.ops.quote_name(cls.tb_table_name), ) vals = [cls.steplen, cls.steplen] cursor.execute(sql, vals) # fix the numchild field vals = ['_' * cls.steplen] # the cake and sql portability are a lie if cls.get_database_vendor('read') == 'mysql': sql = "SELECT tbn1.path, tbn1.numchild, ("\ "SELECT COUNT(1) "\ "FROM %(table)s AS tbn2 "\ "WHERE tbn2.path LIKE "\ "CONCAT(tbn1.path, %%s)) AS real_numchild "\ "FROM %(table)s AS tbn1 "\ "HAVING tbn1.numchild != real_numchild" % { 'table': connection.ops.quote_name( cls.tb_table_name)} else: subquery = "(SELECT COUNT(1) FROM %(table)s AS tbn2"\ " WHERE tbn2.path LIKE tbn1.path||%%s)" sql = ("SELECT tbn1.path, tbn1.numchild, " + subquery + " FROM %(table)s AS tbn1 WHERE tbn1.numchild != " + subquery) sql = sql % { 'table': connection.ops.quote_name(cls.tb_table_name)} # we include the subquery twice vals *= 2 cursor.execute(sql, vals) sql = "UPDATE %(table)s "\ "SET numchild=%%s "\ "WHERE path=%%s" % { 'table': connection.ops.quote_name(cls.tb_table_name)} for node_data in cursor.fetchall(): vals = [node_data[2], node_data[0]] cursor.execute(sql, vals) transaction.commit_unless_managed()
def SetupQueueThread( tid, run, probe_servers, progress_queue, ): connection.close() try: while True: item = probe_servers.get() if isinstance(item, int): try: item = Probe.Server.objects.get(id=item) except: probe_servers.task_done() continue hostname = item.servername.strip() if (not hostname or any([ x in hostname for x in " \t%/&#\"'\\{[]}()*,;<>$" ]) or any([ord(x) >= 128 or ord(x) <= 32 for x in hostname])): item.enabled = False item.save() probe_servers.task_done() continue hostname = hostname.strip(".") while hostname.find("..") >= 0: hostname = hostname.replace("..", ".") if hostname != item.servername: item.enabled = False item.save() item = "0," + hostname + "," + str( item.port ) + "," + item.protocol # Convert to string to correct the list if not isinstance(item, Probe.Server): hostname_line = item if not hostname_line.strip(): probe_servers.task_done() continue split_line = hostname_line.strip().split(",") if len(split_line) > 3: (index, hostname, port, protocol) = split_line[:4] port = port.strip() protocol = protocol.strip() if not protocol or protocol.upper( ) not in Probe.Server.PROTOCOL_PORT_MAP: protocol = Probe.Server.PROTOCOL_PORT.get( int(port) if port else 443, Probe.Server.PROTOCOL_HTTPS) elif len(split_line) > 2: (index, hostname, port) = split_line[:3] port = port.strip() protocol = Probe.Server.PROTOCOL_PORT.get( int(port) if port else 443, Probe.Server.PROTOCOL_HTTPS) else: port = "" (var1, var2) = split_line if var1.isdigit(): (index, hostname) = (var1, var2) else: (hostname, port) = (var1, var2) port = port.strip() protocol = Probe.Server.PROTOCOL_PORT.get( int(port) if port else 443, Probe.Server.PROTOCOL_HTTPS) hostname = hostname.strip() if (not hostname or any([ x in hostname for x in " \t%/&#\"'\\{[]}()*,;<>$" ]) or any([ord(x) >= 128 or ord(x) <= 32 for x in hostname])): probe_servers.task_done() continue hostname = hostname.strip(".") while hostname.find("..") >= 0: hostname = hostname.replace("..", ".") if not port: port = 443 else: port = int(port) sn_t = "%s:%05d:%s" % (hostname, port, protocol) (item, created) = Probe.Server.objects.get_or_create( full_servername=sn_t, defaults={ 'enabled': True, "alexa_rating": 0, "servername": hostname, "port": port, "protocol": protocol.upper(), }) if created: item.Construct() if item.enabled: try: #run_entry = Probe.ProbeQueue.objects.create(part_of_run=run,server=item,state=Probe.ProbeQueue.PROBEQ_IDLE) sid = transaction.savepoint() cursor = connection.cursor() cursor.execute( """INSERT INTO probedata2_probequeue (part_of_run_id, server_id, state) VALUES (%s,%s,%s)""", [ str(run.id), str(item.id), Probe.ProbeQueue.PROBEQ_IDLE ]) transaction.savepoint_commit(sid) transaction.commit_unless_managed() progress_queue.put(True) except: transaction.savepoint_rollback(sid) pass probe_servers.task_done() except: pass
def save_base(self, raw=False, cls=None, origin=None, force_insert=False, force_update=False, using=None): """ Does the heavy-lifting involved in saving. Subclasses shouldn't need to override this method. It's separate from save() in order to hide the need for overrides of save() to pass around internal-only parameters ('raw', 'cls', and 'origin'). """ using = using or router.db_for_write(self.__class__, instance=self) assert not (force_insert and force_update) if cls is None: cls = self.__class__ meta = cls._meta if not meta.proxy: origin = cls else: meta = cls._meta if origin and not meta.auto_created: signals.pre_save.send(sender=origin, instance=self, raw=raw, using=using) # If we are in a raw save, save the object exactly as presented. # That means that we don't try to be smart about saving attributes # that might have come from the parent class - we just save the # attributes we have been given to the class we have been given. # We also go through this process to defer the save of proxy objects # to their actual underlying model. if not raw or meta.proxy: if meta.proxy: org = cls else: org = None for parent, field in meta.parents.items(): # At this point, parent's primary key field may be unknown # (for example, from administration form which doesn't fill # this field). If so, fill it. if field and getattr( self, parent._meta.pk.attname) is None and getattr( self, field.attname) is not None: setattr(self, parent._meta.pk.attname, getattr(self, field.attname)) self.save_base(cls=parent, origin=org, using=using) if field: setattr(self, field.attname, self._get_pk_val(parent._meta)) if meta.proxy: return if not meta.proxy: non_pks = [f for f in meta.local_fields if not f.primary_key] # First, try an UPDATE. If that doesn't update anything, do an INSERT. pk_val = self._get_pk_val(meta) pk_set = pk_val is not None record_exists = True manager = cls._base_manager if pk_set: # Determine whether a record with the primary key already exists. if (force_update or (not force_insert and manager.using(using).filter(pk=pk_val).exists())): # It does already exist, so do an UPDATE. if force_update or non_pks: values = [(f, None, (raw and getattr(self, f.attname) or f.pre_save(self, False))) for f in non_pks] if values: rows = manager.using(using).filter( pk=pk_val)._update(values) if force_update and not rows: raise DatabaseError( "Forced update did not affect any rows.") else: record_exists = False if not pk_set or not record_exists: if meta.order_with_respect_to: # If this is a model with an order_with_respect_to # autopopulate the _order field field = meta.order_with_respect_to order_value = manager.using(using).filter( **{ field.name: getattr(self, field.attname) }).count() self._order = order_value fields = meta.local_fields if not pk_set: if force_update: raise ValueError( "Cannot force an update in save() with no primary key." ) fields = [ f for f in fields if not isinstance(f, AutoField) ] record_exists = False update_pk = bool(meta.has_auto_field and not pk_set) result = manager._insert([self], fields=fields, return_id=update_pk, using=using, raw=raw) if update_pk: setattr(self, meta.pk.attname, result) transaction.commit_unless_managed(using=using) # Store the database on which the object was saved self._state.db = using # Once saved, this is no longer a to-be-added instance. self._state.adding = False # Signal that the save is complete if origin and not meta.auto_created: signals.post_save.send(sender=origin, instance=self, created=(not record_exists), raw=raw, using=using)
return event.group = group for view in views: group.views.add(view) # save the event unless its been sampled if not is_sample: try: event.save() except IntegrityError: transaction.rollback_unless_managed(using=group._state.db) return event transaction.commit_unless_managed(using=group._state.db) if settings.USE_SEARCH: try: maybe_delay(index_event, event) except Exception, e: transaction.rollback_unless_managed(using=group._state.db) logger.exception(u'Error indexing document: %s', e) if is_new: try: regression_signal.send(sender=self.model, instance=group) except Exception, e: transaction.rollback_unless_managed(using=group._state.db) logger.exception(u'Error sending regression signal: %s', e)
def delete(self, key): cursor = connection.cursor() cursor.execute("DELETE FROM %s WHERE cache_key = %%s" % self._table, [key]) transaction.commit_unless_managed()
def handle(self, *fixture_labels, **options): using = options.get('database', DEFAULT_DB_ALIAS) connection = connections[using] self.style = no_style() verbosity = int(options.get('verbosity', 1)) show_traceback = options.get('traceback', False) # commit is a stealth option - it isn't really useful as # a command line option, but it can be useful when invoking # loaddata from within another script. # If commit=True, loaddata will use its own transaction; # if commit=False, the data load SQL will become part of # the transaction in place when loaddata was invoked. commit = options.get('commit', True) # Keep a count of the installed objects and fixtures fixture_count = 0 loaded_object_count = 0 fixture_object_count = 0 models = set() humanize = lambda dirname: dirname and "'%s'" % dirname or 'absolute path' # Get a cursor (even though we don't need one yet). This has # the side effect of initializing the test database (if # it isn't already initialized). cursor = connection.cursor() # Start transaction management. All fixtures are installed in a # single transaction to ensure that all references are resolved. if commit: transaction.commit_unless_managed(using=using) transaction.enter_transaction_management(using=using) transaction.managed(True, using=using) class SingleZipReader(zipfile.ZipFile): def __init__(self, *args, **kwargs): zipfile.ZipFile.__init__(self, *args, **kwargs) if settings.DEBUG: assert len( self.namelist() ) == 1, "Zip-compressed fixtures must contain only one file." def read(self): return zipfile.ZipFile.read(self, self.namelist()[0]) compression_types = { None: file, 'gz': gzip.GzipFile, 'zip': SingleZipReader } if has_bz2: compression_types['bz2'] = bz2.BZ2File app_module_paths = [] for app in get_apps(): if hasattr(app, '__path__'): # It's a 'models/' subpackage for path in app.__path__: app_module_paths.append(path) else: # It's a models.py module app_module_paths.append(app.__file__) app_fixtures = [ os.path.join(os.path.dirname(path), 'fixtures') for path in app_module_paths ] for fixture_label in fixture_labels: parts = fixture_label.split('.') if len(parts) > 1 and parts[-1] in compression_types: compression_formats = [parts[-1]] parts = parts[:-1] else: compression_formats = compression_types.keys() if len(parts) == 1: fixture_name = parts[0] formats = serializers.get_public_serializer_formats() else: fixture_name, format = '.'.join(parts[:-1]), parts[-1] if format in serializers.get_public_serializer_formats(): formats = [format] else: formats = [] if formats: if verbosity > 1: self.stdout.write("Loading '%s' fixtures...\n" % fixture_name) else: self.stderr.write( self.style.ERROR( "Problem installing fixture '%s': %s is not a known serialization format.\n" % (fixture_name, format))) if commit: transaction.rollback(using=using) transaction.leave_transaction_management(using=using) return if os.path.isabs(fixture_name): fixture_dirs = [fixture_name] else: fixture_dirs = app_fixtures + list( settings.FIXTURE_DIRS) + [''] for fixture_dir in fixture_dirs: if verbosity > 1: self.stdout.write("Checking %s for fixtures...\n" % humanize(fixture_dir)) label_found = False for combo in product([using, None], formats, compression_formats): database, format, compression_format = combo file_name = '.'.join( p for p in [fixture_name, database, format, compression_format] if p) if verbosity > 1: self.stdout.write("Trying %s for %s fixture '%s'...\n" % \ (humanize(fixture_dir), file_name, fixture_name)) full_path = os.path.join(fixture_dir, file_name) open_method = compression_types[compression_format] try: fixture = open_method(full_path, 'r') if label_found: fixture.close() self.stderr.write( self.style.ERROR( "Multiple fixtures named '%s' in %s. Aborting.\n" % (fixture_name, humanize(fixture_dir)))) if commit: transaction.rollback(using=using) transaction.leave_transaction_management( using=using) return else: fixture_count += 1 objects_in_fixture = 0 loaded_objects_in_fixture = 0 if verbosity > 0: self.stdout.write("Installing %s fixture '%s' from %s.\n" % \ (format, fixture_name, humanize(fixture_dir))) try: objects = serializers.deserialize(format, fixture, using=using) for obj in objects: objects_in_fixture += 1 if router.allow_syncdb( using, obj.object.__class__): loaded_objects_in_fixture += 1 models.add(obj.object.__class__) obj.save(using=using) loaded_object_count += loaded_objects_in_fixture fixture_object_count += objects_in_fixture label_found = True except (SystemExit, KeyboardInterrupt): raise except Exception: import traceback fixture.close() if commit: transaction.rollback(using=using) transaction.leave_transaction_management( using=using) if show_traceback: traceback.print_exc() else: self.stderr.write( self.style.ERROR( "Problem installing fixture '%s': %s\n" % (full_path, ''.join( traceback.format_exception( sys.exc_type, sys.exc_value, sys.exc_traceback))))) return fixture.close() # If the fixture we loaded contains 0 objects, assume that an # error was encountered during fixture loading. if objects_in_fixture == 0: self.stderr.write( self.style.ERROR( "No fixture data found for '%s'. (File format may be invalid.)\n" % (fixture_name))) if commit: transaction.rollback(using=using) transaction.leave_transaction_management( using=using) return except Exception, e: if verbosity > 1: self.stdout.write("No %s fixture '%s' in %s.\n" % \ (format, fixture_name, humanize(fixture_dir)))
def handle(self, *args, **options): transaction.commit_unless_managed() transaction.enter_transaction_management() transaction.managed(True) site = Site.objects.all()[0] cursor = connection.cursor() cursor.execute(FREE_SQL) for row in cursor: (content_type_id, object_id, parent_id, name, website, email, date_submitted, date_modified, date_approved, comment, markup, is_public, is_approved, ip_address) = row tc = ThreadedComment( content_type_id=content_type_id, object_pk=object_id, user_name=name, user_email=email, user_url=website, comment=comment, submit_date=date_submitted, ip_address=ip_address, is_public=is_public, is_removed=not is_approved, parent_id=parent_id, site=site, ) tc.save(skip_tree_path=True) cursor = connection.cursor() cursor.execute(USER_SQL) for row in cursor: (content_type_id, object_id, parent_id, user_id, date_submitted, date_modified, date_approved, comment, markup, is_public, is_approved, ip_address) = row tc = ThreadedComment( content_type_id=content_type_id, object_pk=object_id, user_id=user_id, comment=comment, submit_date=date_submitted, ip_address=ip_address, is_public=is_public, is_removed=not is_approved, parent_id=parent_id, site=site, ) tc.save(skip_tree_path=True) for comment in ThreadedComment.objects.all(): path = [str(comment.id).zfill(PATH_DIGITS)] current = comment while current.parent: current = current.parent path.append(str(current.id).zfill(PATH_DIGITS)) comment.tree_path = PATH_SEPARATOR.join(reversed(path)) comment.save(skip_tree_path=True) if comment.parent: ThreadedComment.objects.filter(pk=comment.parent.pk).update( last_child=comment) transaction.commit() transaction.leave_transaction_management()
else: transaction.commit_unless_managed() else: if verbosity >= 2: print "No custom SQL for %s.%s model" % ( app_name, model._meta.object_name) # Install SQL indicies for all newly created models for app in models.get_apps(): app_name = app.__name__.split('.')[-2] for model in models.get_models(app): if model in created_models: index_sql = connection.creation.sql_indexes_for_model( model, self.style) if index_sql: if verbosity >= 1: print "Installing index for %s.%s model" % ( app_name, model._meta.object_name) try: for sql in index_sql: cursor.execute(sql) except Exception, e: sys.stderr.write("Failed to install index for %s.%s model: %s\n" % \ (app_name, model._meta.object_name, e)) transaction.rollback_unless_managed() else: transaction.commit_unless_managed() # Install the 'initial_data' fixture, using format discovery from django.core.management import call_command call_command('loaddata', 'initial_data', verbosity=verbosity)
def handle_noargs(self, **options): verbosity = int(options.get('verbosity')) interactive = options.get('interactive') show_traceback = options.get('traceback') load_initial_data = options.get('load_initial_data') self.style = no_style() # Import the 'management' module within each installed app, to register # dispatcher events. for app_name in settings.INSTALLED_APPS: try: import_module('.management', app_name) except ImportError as exc: # This is slightly hackish. We want to ignore ImportErrors # if the "management" module itself is missing -- but we don't # want to ignore the exception if the management module exists # but raises an ImportError for some reason. The only way we # can do this is to check the text of the exception. Note that # we're a bit broad in how we check the text, because different # Python implementations may not use the same text. # CPython uses the text "No module named management" # PyPy uses "No module named myproject.myapp.management" msg = exc.args[0] if not msg.startswith( 'No module named') or 'management' not in msg: raise db = options.get('database') connection = connections[db] cursor = connection.cursor() # Get a list of already installed *models* so that references work right. tables = connection.introspection.table_names() seen_models = connection.introspection.installed_models(tables) created_models = set() pending_references = {} # Build the manifest of apps and models that are to be synchronized all_models = [(app.__name__.split('.')[-2], [ m for m in models.get_models(app, include_auto_created=True) if router.allow_syncdb(db, m) ]) for app in models.get_apps()] def model_installed(model): opts = model._meta converter = connection.introspection.table_name_converter return not ((converter(opts.db_table) in tables) or (opts.auto_created and converter( opts.auto_created._meta.db_table) in tables)) manifest = SortedDict( (app_name, list(filter(model_installed, model_list))) for app_name, model_list in all_models) # Create the tables for each model if verbosity >= 1: self.stdout.write("Creating tables ...\n") for app_name, model_list in manifest.items(): for model in model_list: # Create the model's database table, if it doesn't already exist. if verbosity >= 3: self.stdout.write("Processing %s.%s model\n" % (app_name, model._meta.object_name)) sql, references = connection.creation.sql_create_model( model, self.style, seen_models) seen_models.add(model) created_models.add(model) for refto, refs in references.items(): pending_references.setdefault(refto, []).extend(refs) if refto in seen_models: sql.extend( connection.creation.sql_for_pending_references( refto, self.style, pending_references)) sql.extend( connection.creation.sql_for_pending_references( model, self.style, pending_references)) if verbosity >= 1 and sql: self.stdout.write("Creating table %s\n" % model._meta.db_table) for statement in sql: cursor.execute(statement) tables.append( connection.introspection.table_name_converter( model._meta.db_table)) transaction.commit_unless_managed(using=db) # Send the post_syncdb signal, so individual apps can do whatever they need # to do at this point. emit_post_sync_signal(created_models, verbosity, interactive, db) # The connection may have been closed by a syncdb handler. cursor = connection.cursor() if verbosity >= 1: self.stdout.write("Installing indexes ...\n") # Install SQL indices for all newly created models for app_name, model_list in manifest.items(): for model in model_list: if model in created_models: index_sql = connection.creation.sql_indexes_for_model( model, self.style) if index_sql: if verbosity >= 2: self.stdout.write( "Installing index for %s.%s model\n" % (app_name, model._meta.object_name)) try: for sql in index_sql: cursor.execute(sql) except Exception as e: self.stderr.write("Failed to install index for %s.%s model: %s\n" % \ (app_name, model._meta.object_name, e)) transaction.rollback_unless_managed(using=db) else: transaction.commit_unless_managed(using=db) # Install custom SQL for the app (but only if this # is a model we've just created) if verbosity >= 1: self.stdout.write("Installing custom SQL ...\n") for app_name, model_list in manifest.items(): for model in model_list: if model in created_models: custom_sql = custom_sql_for_model(model, self.style, connection) if custom_sql: if verbosity >= 2: self.stdout.write( "Installing custom SQL for %s.%s model\n" % (app_name, model._meta.object_name)) try: for sql in custom_sql: cursor.execute(sql) except Exception as e: self.stderr.write("Failed to install custom SQL for %s.%s model: %s\n" % \ (app_name, model._meta.object_name, e)) if show_traceback: traceback.print_exc() transaction.rollback_unless_managed(using=db) else: transaction.commit_unless_managed(using=db) else: if verbosity >= 3: self.stdout.write( "No custom SQL for %s.%s model\n" % (app_name, model._meta.object_name)) # Load initial_data fixtures (unless that has been disabled) if load_initial_data: call_command('loaddata', 'initial_data', verbosity=verbosity, database=db, skip_validation=True)
class Command(NoArgsCommand): option_list = NoArgsCommand.option_list + (make_option( '--noinput', action='store_false', dest='interactive', default=True, help='Tells Django to NOT prompt the user for input of any kind.'), ) help = "Create the database tables for all apps in INSTALLED_APPS whose tables haven't already been created." def handle_noargs(self, **options): from django.db import connection, transaction, models from django.conf import settings from django.core.management.sql import custom_sql_for_model, emit_post_sync_signal verbosity = int(options.get('verbosity', 1)) interactive = options.get('interactive') show_traceback = options.get('traceback', False) self.style = no_style() # Import the 'management' module within each installed app, to register # dispatcher events. for app_name in settings.INSTALLED_APPS: try: __import__(app_name + '.management', {}, {}, ['']) except ImportError, exc: # This is slightly hackish. We want to ignore ImportErrors # if the "management" module itself is missing -- but we don't # want to ignore the exception if the management module exists # but raises an ImportError for some reason. The only way we # can do this is to check the text of the exception. Note that # we're a bit broad in how we check the text, because different # Python implementations may not use the same text. # CPython uses the text "No module named management" # PyPy uses "No module named myproject.myapp.management" msg = exc.args[0] if not msg.startswith( 'No module named') or 'management' not in msg: raise cursor = connection.cursor() # Get a list of already installed *models* so that references work right. tables = connection.introspection.table_names() seen_models = connection.introspection.installed_models(tables) created_models = set() pending_references = {} # Create the tables for each model for app in models.get_apps(): app_name = app.__name__.split('.')[-2] model_list = models.get_models(app) for model in model_list: # Create the model's database table, if it doesn't already exist. if verbosity >= 2: print "Processing %s.%s model" % (app_name, model._meta.object_name) if connection.introspection.table_name_converter( model._meta.db_table) in tables: continue sql, references = connection.creation.sql_create_model( model, self.style, seen_models) seen_models.add(model) created_models.add(model) for refto, refs in references.items(): pending_references.setdefault(refto, []).extend(refs) if refto in seen_models: sql.extend( connection.creation.sql_for_pending_references( refto, self.style, pending_references)) sql.extend( connection.creation.sql_for_pending_references( model, self.style, pending_references)) if verbosity >= 1: print "Creating table %s" % model._meta.db_table for statement in sql: cursor.execute(statement) tables.append( connection.introspection.table_name_converter( model._meta.db_table)) # Create the m2m tables. This must be done after all tables have been created # to ensure that all referred tables will exist. for app in models.get_apps(): app_name = app.__name__.split('.')[-2] model_list = models.get_models(app) for model in model_list: if model in created_models: sql = connection.creation.sql_for_many_to_many( model, self.style) if sql: if verbosity >= 2: print "Creating many-to-many tables for %s.%s model" % ( app_name, model._meta.object_name) for statement in sql: cursor.execute(statement) transaction.commit_unless_managed() # Send the post_syncdb signal, so individual apps can do whatever they need # to do at this point. emit_post_sync_signal(created_models, verbosity, interactive) # The connection may have been closed by a syncdb handler. cursor = connection.cursor() # Install custom SQL for the app (but only if this # is a model we've just created) for app in models.get_apps(): app_name = app.__name__.split('.')[-2] for model in models.get_models(app): if model in created_models: custom_sql = custom_sql_for_model(model, self.style) if custom_sql: if verbosity >= 1: print "Installing custom SQL for %s.%s model" % ( app_name, model._meta.object_name) try: for sql in custom_sql: cursor.execute(sql) except Exception, e: sys.stderr.write("Failed to install custom SQL for %s.%s model: %s\n" % \ (app_name, model._meta.object_name, e)) if show_traceback: import traceback traceback.print_exc() transaction.rollback_unless_managed() else: transaction.commit_unless_managed() else: if verbosity >= 2: print "No custom SQL for %s.%s model" % ( app_name, model._meta.object_name)
def delete(request,*kw,**kwargs): if request.method == 'GET': form = forms.DeleteForm() return django.shortcuts.render_to_response('bots/delete.html', {'form': form},context_instance=django.template.RequestContext(request)) else: if 'submit' in request.POST: form = forms.DeleteForm(request.POST) if form.is_valid(): botsglobal.logger.info(_(u'Start deleting in configuration.')) if form.cleaned_data['deltransactions']: #while testing with very big loads, deleting transaction when wrong. Using raw SQL solved this. from django.db import connection, transaction cursor = connection.cursor() cursor.execute("DELETE FROM ta") cursor.execute("DELETE FROM filereport") cursor.execute("DELETE FROM report") transaction.commit_unless_managed() request.user.message_set.create(message=_(u'Transactions are deleted.')) botsglobal.logger.info(_(u' Transactions are deleted.')) #clean data files deletefrompath = botsglobal.ini.get('directories','data','botssys/data') shutil.rmtree(deletefrompath,ignore_errors=True) botslib.dirshouldbethere(deletefrompath) request.user.message_set.create(message=_(u'Data files are deleted.')) botsglobal.logger.info(_(u' Data files are deleted.')) if form.cleaned_data['delconfiguration']: models.confirmrule.objects.all().delete() models.channel.objects.all().delete() models.chanpar.objects.all().delete() models.partner.objects.all().delete() models.translate.objects.all().delete() models.routes.objects.all().delete() request.user.message_set.create(message=_(u'Database configuration is deleted.')) botsglobal.logger.info(_(u' Database configuration is deleted.')) if form.cleaned_data['delcodelists']: models.ccode.objects.all().delete() models.ccodetrigger.objects.all().delete() request.user.message_set.create(message=_(u'User code lists are deleted.')) botsglobal.logger.info(_(u' User code lists are deleted.')) if form.cleaned_data['delinfile']: deletefrompath = botslib.join(botsglobal.ini.get('directories','botssys','botssys'),'infile') shutil.rmtree('bots/botssys/infile',ignore_errors=True) request.user.message_set.create(message=_(u'Files in botssys/infile are deleted.')) botsglobal.logger.info(_(u' Files in botssys/infile are deleted.')) if form.cleaned_data['deloutfile']: deletefrompath = botslib.join(botsglobal.ini.get('directories','botssys','botssys'),'outfile') shutil.rmtree('bots/botssys/outfile',ignore_errors=True) request.user.message_set.create(message=_(u'Files in botssys/outfile are deleted.')) botsglobal.logger.info(_(u' Files in botssys/outfile are deleted.')) if form.cleaned_data['deluserscripts']: deletefrompath = botsglobal.ini.get('directories','usersysabs') for root, dirs, files in os.walk(deletefrompath): head, tail = os.path.split(root) if tail == 'charsets': del dirs[:] continue for bestand in files: if bestand != '__init__.py': os.remove(os.path.join(root,bestand)) request.user.message_set.create(message=_(u'User scripts are deleted (in usersys).')) botsglobal.logger.info(_(u' User scripts are deleted (in usersys).')) elif form.cleaned_data['delbackup']: deletefrompath = botsglobal.ini.get('directories','usersysabs') for root, dirs, files in os.walk(deletefrompath): head, tail = os.path.split(root) if tail == 'charsets': del dirs[:] continue for bestand in files: name,ext = os.path.splitext(bestand) if ext and len(ext) == 15 and ext[1:].isdigit() : os.remove(os.path.join(root,bestand)) request.user.message_set.create(message=_(u'Backupped user scripts are deleted/purged (in usersys).')) botsglobal.logger.info(_(u' Backupped user scripts are deleted/purged (in usersys).')) botsglobal.logger.info(_(u'Finished deleting in configuration.')) return django.shortcuts.redirect('/home')
def load_app(app_path): testapp = django_load_app(app_path) app_name = testapp.__name__.split('.')[-2] connection = connections[DEFAULT_DB_ALIAS] cursor = connection.cursor() test_models = [ m for m in models.get_models(testapp, include_auto_created=True) if router.allow_syncdb(DEFAULT_DB_ALIAS, m) ] loaded_models[app_path] = test_models # We assume the models haven't been installed, otherwise there's more to do here # Get a list of already installed *models* so that references work right. tables = connection.introspection.table_names() seen_models = connection.introspection.installed_models(tables) pending_references = {} verbosity = 0 # Create the tables for each model for model in test_models: # Create the model's database table, if it doesn't already exist. if verbosity >= 2: print "Processing %s.%s model" % (app_name, model._meta.object_name) sql, references = connection.creation.sql_create_model( model, no_style(), seen_models) seen_models.add(model) for refto, refs in references.items(): pending_references.setdefault(refto, []).extend(refs) if refto in seen_models: sql.extend( connection.creation.sql_for_pending_references( refto, no_style(), pending_references)) sql.extend( connection.creation.sql_for_pending_references( model, no_style(), pending_references)) if verbosity >= 1 and sql: print "Creating table %s" % model._meta.db_table for statement in sql: cursor.execute(statement) tables.append( connection.introspection.table_name_converter( model._meta.db_table)) transaction.commit_unless_managed(using=DEFAULT_DB_ALIAS) for model in test_models: index_sql = connection.creation.sql_indexes_for_model( model, no_style()) if index_sql: if verbosity >= 1: print "Installing index for %s.%s model" % ( app_name, model._meta.object_name) try: for sql in index_sql: cursor.execute(sql) except Exception, e: sys.stderr.write("Failed to install index for %s.%s model: %s\n" % \ (app_name, model._meta.object_name, e)) transaction.rollback_unless_managed(using=DEFAULT_DB_ALIAS) else: transaction.commit_unless_managed(using=DEFAULT_DB_ALIAS)
def handle(self, *fixture_labels, **options): """ Main method of a Django command """ from django.db.models import get_apps from django.core import serializers from django.db import connection, transaction from django.conf import settings self.style = no_style() verbosity = int(options.get('verbosity', 1)) show_traceback = options.get('traceback', False) # Keep a count of the installed objects and fixtures fixture_count = 0 object_count = 0 objects_per_fixture = [] models = set() humanize = lambda dirname: dirname and "'%s'" % dirname or 'absolute path' # Get a cursor (even though we don't need one yet). This has # the side effect of initializing the test database (if # it isn't already initialized). cursor = connection.cursor() # Start transaction management. All fixtures are installed in a # single transaction to ensure that all references are resolved. transaction.commit_unless_managed() transaction.enter_transaction_management() transaction.managed(True) app_fixtures = [os.path.join(os.path.dirname(app.__file__), 'fixtures') \ for app in get_apps()] for fixture_label in fixture_labels: parts = fixture_label.split('.') if len(parts) == 1: fixture_name = fixture_label formats = serializers.get_public_serializer_formats() else: fixture_name, format = '.'.join(parts[:-1]), parts[-1] if format in serializers.get_public_serializer_formats(): formats = [format] else: formats = [] if formats: if verbosity > 1: print "Loading '%s' fixtures..." % fixture_name else: sys.stderr.write( self.style.ERROR("Problem installing fixture '%s': %s is not a known " + \ "serialization format." % (fixture_name, format)) ) transaction.rollback() transaction.leave_transaction_management() return if os.path.isabs(fixture_name): fixture_dirs = [fixture_name] else: fixture_dirs = app_fixtures + list( settings.FIXTURE_DIRS) + [''] for fixture_dir in fixture_dirs: if verbosity > 1: print "Checking %s for fixtures..." % humanize(fixture_dir) label_found = False for format in formats: serializer = serializers.get_serializer(format) if verbosity > 1: print "Trying %s for %s fixture '%s'..." % \ (humanize(fixture_dir), format, fixture_name) try: full_path = os.path.join( fixture_dir, '.'.join([fixture_name, format])) fixture = open(full_path, 'r') if label_found: fixture.close() print self.style.ERROR( "Multiple fixtures named '%s' in %s. Aborting." % (fixture_name, humanize(fixture_dir))) transaction.rollback() transaction.leave_transaction_management() return else: fixture_count += 1 objects_per_fixture.append(0) if verbosity > 0: print "Installing %s fixture '%s' from %s." % \ (format, fixture_name, humanize(fixture_dir)) try: objects_to_keep = {} objects = serializers.deserialize( format, fixture) for obj in objects: object_count += 1 objects_per_fixture[-1] += 1 class_ = obj.object.__class__ if not class_ in objects_to_keep: objects_to_keep[class_] = set() objects_to_keep[class_].add(obj.object) models.add(class_) obj.save() self.remove_objects_not_in( objects_to_keep, verbosity) label_found = True except (SystemExit, KeyboardInterrupt): raise except Exception: import traceback fixture.close() transaction.rollback() transaction.leave_transaction_management() if show_traceback: traceback.print_exc() else: sys.stderr.write( self.style.ERROR( "Problem installing fixture '%s': %s\n" % (full_path, traceback.format_exc()))) return fixture.close() except: if verbosity > 1: print "No %s fixture '%s' in %s." % \ (format, fixture_name, humanize(fixture_dir)) # If any of the fixtures we loaded contain 0 objects, assume that an # error was encountered during fixture loading. if 0 in objects_per_fixture: sys.stderr.write( self.style.ERROR( "No fixture data found for '%s'. (File format may be invalid.)" % (fixture_name))) transaction.rollback() transaction.leave_transaction_management() return # If we found even one object in a fixture, we need to reset the # database sequences. if object_count > 0: sequence_sql = connection.ops.sequence_reset_sql( self.style, models) if sequence_sql: if verbosity > 1: print "Resetting sequences" for line in sequence_sql: cursor.execute(line) transaction.commit() transaction.leave_transaction_management() if object_count == 0: if verbosity > 1: print "No fixtures found." else: if verbosity > 0: print "Installed %d object(s) from %d fixture(s)" % ( object_count, fixture_count) # Close the DB connection. This is required as a workaround for an # edge case in MySQL: if the same connection is used to # create tables, load data, and query, the query can return # incorrect results. See Django #7572, MySQL #37735. connection.close()