def delete_expired(self, expires):
"""Delete all expired taskset results."""
self.get_all_expired(expires).update(hidden=True)
cursor = self.connection_for_write().cursor()
cursor.execute("DELETE FROM %s WHERE hidden=%%s" % (
self.model._meta.db_table, ), (True, ))
transaction.commit_unless_managed()
def delete_objects(seen_objs):
"""
Iterate through a list of seen classes, and remove any instances that are
referred to.
"""
try:
ordered_classes = seen_objs.keys()
except CyclicDependency:
# If there is a cyclic dependency, we cannot in general delete the
# objects. However, if an appropriate transaction is set up, or if the
# database is lax enough, it will succeed. So for now, we go ahead and
# try anyway.
ordered_classes = seen_objs.unordered_keys()
obj_pairs = {}
for cls in ordered_classes:
items = seen_objs[cls].items()
items.sort()
obj_pairs[cls] = items
# Pre-notify all instances to be deleted.
for pk_val, instance in items:
signals.pre_delete.send(sender=cls, instance=instance)
pk_list = [pk for pk,instance in items]
del_query = sql.DeleteQuery(cls, connection)
del_query.delete_batch_related(pk_list)
update_query = sql.UpdateQuery(cls, connection)
for field, model in cls._meta.get_fields_with_model():
if (field.rel and field.null and field.rel.to in seen_objs and
filter(lambda f: f.column == field.column,
field.rel.to._meta.fields)):
if model:
sql.UpdateQuery(model, connection).clear_related(field,
pk_list)
else:
update_query.clear_related(field, pk_list)
# Now delete the actual data.
for cls in ordered_classes:
items = obj_pairs[cls]
items.reverse()
pk_list = [pk for pk,instance in items]
del_query = sql.DeleteQuery(cls, connection)
del_query.delete_batch(pk_list)
# Last cleanup; set NULLs where there once was a reference to the
# object, NULL the primary key of the found objects, and perform
# post-notification.
for pk_val, instance in items:
for field in cls._meta.fields:
if field.rel and field.null and field.rel.to in seen_objs:
setattr(instance, field.attname, None)
signals.post_delete.send(sender=cls, instance=instance)
setattr(instance, cls._meta.pk.attname, None)
transaction.commit_unless_managed()
def add_root(cls, **kwargs):
"Adds a root node to the tree."
# do we have a root node already?
last_root = cls.get_last_root_node()
if last_root and last_root.node_order_by:
# there are root nodes and node_order_by has been set
# delegate sorted insertion to add_sibling
return last_root.add_sibling('sorted-sibling', **kwargs)
if last_root:
# adding the new root node as the last one
newtree_id = last_root.tree_id + 1
else:
# adding the first root node
newtree_id = 1
# creating the new object
newobj = cls(**kwargs)
newobj.depth = 1
newobj.tree_id = newtree_id
newobj.lft = 1
newobj.rgt = 2
# saving the instance before returning it
newobj.save()
transaction.commit_unless_managed()
return newobj
def migrate_media_field_forwards(model, image_field):
cursor = connection.cursor()
cursor.execute(
"UPDATE %s SET %s = SUBSTR(%s, 7) WHERE %s LIKE 'media%%%%'" %
(model._meta.db_table, image_field, image_field, image_field)
)
transaction.commit_unless_managed()
def backwards(self, orm):
from django.db import connection, transaction
cursor = connection.cursor()
cursor.execute("UPDATE music_album SET cover = REPLACE(cover, 'dynamic/media', 'media');")
transaction.commit_unless_managed()
cursor.execute("UPDATE music_song SET tab = REPLACE(tab, 'dynamic/media', 'media');")
transaction.commit_unless_managed()
def _queue_requests():
""" Add tableau factory generation requests for each location and factory.
We add a record to the TableauFactoryGenerationRequest table for each
location and tableau factory that we want to generate. The daemon will
then pick up the requests and fire them off, one at a time, to our
private API call.
"""
# Ask the factory generator to assemble a list of all the tableau-creation
# requests.
requests = factory_generator.calc_generation_requests()
# We now have a single large list of requests. These might be too big to
# add to the database all at once, so break them into chunks and add them
# one chunk at a time.
cursor = connections['default'].cursor()
for chunk in _splitList(requests, 1000):
cmd = "INSERT INTO tableau_editor_tableaufactorygenerationrequest " \
+ "(factory_id, loc_code) VALUES (%s, %s)"
if len(chunk) == 1:
cursor.execute(cmd, chunk[0])
else:
cursor.executemany(cmd, chunk)
transaction.commit_unless_managed() # Commit changes after each chunk.
def add_root(cls, **kwargs):
"""
Adds a root node to the tree.
:raise PathOverflow: when no more root objects can be added
"""
# do we have a root node already?
last_root = cls.get_last_root_node()
if last_root and last_root.node_order_by:
# there are root nodes and node_order_by has been set
# delegate sorted insertion to add_sibling
return last_root.add_sibling('sorted-sibling', **kwargs)
if last_root:
# adding the new root node as the last one
newpath = cls._inc_path(last_root.path)
else:
# adding the first root node
newpath = cls._get_path(None, 1, 1)
# creating the new object
newobj = cls(**kwargs)
newobj.depth = 1
newobj.path = newpath
# saving the instance before returning it
newobj.save()
transaction.commit_unless_managed()
return newobj
def handle_noargs(self, **options):
"""
Command execution.
"""
self.cursor = connection.cursor()
self.introspection = connection.introspection
found_missing_fields = False
models = translator.get_registered_models(abstract=False)
for model in models:
db_table = model._meta.db_table
model_full_name = '%s.%s' % (model._meta.app_label, model._meta.module_name)
opts = translator.get_options_for_model(model)
for field_name in opts.local_fields.iterkeys():
missing_langs = list(self.get_missing_languages(field_name, db_table))
if missing_langs:
found_missing_fields = True
print_missing_langs(missing_langs, field_name, model_full_name)
sql_sentences = self.get_sync_sql(field_name, missing_langs, model)
execute_sql = ask_for_confirmation(sql_sentences, model_full_name)
if execute_sql:
print 'Executing SQL...',
for sentence in sql_sentences:
self.cursor.execute(sentence)
print 'Done'
else:
print 'SQL not executed'
transaction.commit_unless_managed()
if not found_missing_fields:
print 'No new translatable fields detected'
def load_bulk(cls, bulk_data, parent=None, keep_ids=False):
"Loads a list/dictionary structure to the tree."
# tree, iterative preorder
added = []
if parent:
parent_id = parent.id
else:
parent_id = None
# stack of nodes to analize
stack = [(parent_id, node) for node in bulk_data[::-1]]
while stack:
parent_id, node_struct = stack.pop()
# shallow copy of the data strucure so it doesn't persist...
node_data = node_struct['data'].copy()
if keep_ids:
node_data['id'] = node_struct['id']
if parent_id:
parent = cls.objects.get(pk=parent_id)
node_obj = parent.add_child(**node_data)
else:
node_obj = cls.add_root(**node_data)
added.append(node_obj.id)
if 'children' in node_struct:
# extending the stack with the current node as the parent of
# the new nodes
stack.extend([(node_obj.id, node) \
for node in node_struct['children'][::-1]])
transaction.commit_unless_managed()
return added
def handle_label(self, tablename, **options):
alias = options.get('database', DEFAULT_DB_ALIAS)
connection = connections[alias]
fields = (
# "key" is a reserved word in MySQL, so use "cache_key" instead.
models.CharField(name='cache_key', max_length=255, unique=True, primary_key=True),
models.TextField(name='value'),
models.DateTimeField(name='expires', db_index=True),
)
table_output = []
index_output = []
qn = connection.ops.quote_name
for f in fields:
field_output = [qn(f.name), f.db_type(connection=connection)]
field_output.append("%sNULL" % (not f.null and "NOT " or ""))
if f.primary_key:
field_output.append("PRIMARY KEY")
elif f.unique:
field_output.append("UNIQUE")
if f.db_index:
unique = f.unique and "UNIQUE " or ""
index_output.append("CREATE %sINDEX %s ON %s (%s);" % \
(unique, qn('%s_%s' % (tablename, f.name)), qn(tablename),
qn(f.name)))
table_output.append(" ".join(field_output))
full_statement = ["CREATE TABLE %s (" % qn(tablename)]
for i, line in enumerate(table_output):
full_statement.append(' %s%s' % (line, i < len(table_output) - 1 and ',' or ''))
full_statement.append(');')
curs = connection.cursor()
curs.execute("\n".join(full_statement))
for statement in index_output:
curs.execute(statement)
transaction.commit_unless_managed(using=alias)
def install(self):
try:
with transaction.atomic():
self.install_atomic()
except AttributeError:
self.install_atomic()
transaction.commit_unless_managed(using=self.using)
def fill_table(sheet, columns, name):
cursor = connection.cursor()
column_sql = """date, player_id, player_surname, player_forename, team, team_id, opposition, opposition_id, venue, """
column_sql += ', '.join(columns)
for row in range(sheet.nrows):
if row == 0:
continue
row_values = [ ]
count = 0
for value in sheet.row_values(row):
if count in varchar_column_indices:
row_values.append("%s" % ('"' + value.encode('utf-8') + '"'))
else:
row_values.append(str(int(value)).encode('utf-8'))
count += 1
values_sql = ','.join(row_values)
sql = 'INSERT INTO %s (%s) VALUES (%s);' % (str(name), str(column_sql), str(values_sql))
# print sql
cursor.execute(sql)
transaction.commit_unless_managed()
def initialize_source_progress(lt_rlz, hzrd_src):
"""
Create ``source_progress`` models for given logic tree realization
and set total sources of realization.
:param lt_rlz:
:class:`openquake.engine.db.models.LtRealization` object to
initialize source progress for.
:param hztd_src:
:class:`openquake.engine.db.models.Input` object that needed parsed
sources are referencing.
"""
cursor = connections["reslt_writer"].cursor()
src_progress_tbl = models.SourceProgress._meta.db_table
parsed_src_tbl = models.ParsedSource._meta.db_table
lt_rlz_tbl = models.LtRealization._meta.db_table
cursor.execute(
"""
INSERT INTO "%s" (lt_realization_id, parsed_source_id, is_complete)
SELECT %%s, id, FALSE
FROM "%s" WHERE input_id = %%s
ORDER BY id
"""
% (src_progress_tbl, parsed_src_tbl),
[lt_rlz.id, hzrd_src.id],
)
cursor.execute(
"""
UPDATE "%s" SET total_items = (
SELECT count(1) FROM "%s" WHERE lt_realization_id = %%s
)"""
% (lt_rlz_tbl, src_progress_tbl),
[lt_rlz.id],
)
transaction.commit_unless_managed()
def wrapped(*args, **kw):
was_exception = False
try:
return fun(*args, **kw)
except:
was_exception = True
# Log the exception so we can actually see it in procuction.
# When celery is upgraded to 2.2, this can be done more
# gracefully with a signal.
# See: http://groups.google.com/group/celery-users/
# browse_thread/thread/95bdffe5a0057ac0?pli=1
exc_info = sys.exc_info()
log.error(u'Celery TASK exception: %s: %s'
% (exc_info[1].__class__.__name__, exc_info[1]),
exc_info=exc_info)
raise
finally:
try:
for db in connections:
transaction.commit_unless_managed(using=db)
except:
if was_exception:
# We want to see the original exception so let's
# just log the one after that.
log.exception(
'While trying to recover from an exception')
else:
raise
def _clear_items(self, source_col_name):
# source_col_name: the PK colname in join_table for the source object
cursor = connection.cursor()
cursor.execute("DELETE FROM %s WHERE %s = %%s" % \
(self.join_table, source_col_name),
[self._pk_val])
transaction.commit_unless_managed()
def handle_noargs(self, **options):
cursor = connection.cursor()
sql_files = glob.glob(os.path.join(settings.MIGRATIONS_ROOT, '*.sql'))
py_files = glob.glob(os.path.join(settings.MIGRATIONS_ROOT, '*.py'))
files = sorted(sql_files + py_files)
alter_db = False
for fn in files:
done = '%s.done' % fn
if not os.path.exists(done):
if fn.endswith('sql'):
f = open(fn, 'r')
sql = f.read()
f.close()
print "Executing: %s" % os.path.split(fn)[1]
cursor.execute(sql)
transaction.commit_unless_managed()
alter_db = True
else:
name, _ = os.path.splitext(os.path.split(fn)[1])
if name.startswith("__"):
continue
__import__('migrations', globals(), locals(), [name])
alter_db = sys.modules['migrations.%s' % name].main()
open(done, 'w').close()
if not alter_db:
print "Up to date, nothing to do."
else:
print "Success!"
def log_answer(answer):
pin_this_thread()
creator = answer.creator
created = answer.created
question = answer.question
users = [a.creator for a in
question.answers.select_related('creator').exclude(
creator=creator)]
if question.creator != creator:
users += [question.creator]
users = set(users) # Remove duplicates.
if users:
action = Action.objects.create(
creator=creator,
created=created,
url=answer.get_absolute_url(),
content_object=answer,
formatter_cls='questions.formatters.AnswerFormatter')
action.users.add(*users)
transaction.commit_unless_managed()
# Record karma actions
AnswerAction(answer.creator, answer.created.date()).save()
if answer == answer.question.answers.order_by('created')[0]:
FirstAnswerAction(answer.creator, answer.created.date()).save()
unpin_this_thread()
def weekly_downloads():
"""
Update 7-day add-on download counts.
"""
cursor = connection.cursor()
cursor.execute("""
SELECT addon_id, SUM(count) AS weekly_count
FROM download_counts
WHERE `date` >= DATE_SUB(CURDATE(), INTERVAL 7 DAY)
GROUP BY addon_id
ORDER BY addon_id""")
counts = cursor.fetchall()
addon_ids = [r[0] for r in counts]
if not addon_ids:
return
cursor.execute("""
SELECT id, 0
FROM addons
WHERE id NOT IN %s""", (addon_ids,))
counts += cursor.fetchall()
cursor.execute("""
CREATE TEMPORARY TABLE tmp
(addon_id INT PRIMARY KEY, count INT)""")
cursor.execute('INSERT INTO tmp VALUES %s' %
','.join(['(%s,%s)'] * len(counts)),
list(itertools.chain(*counts)))
cursor.execute("""
UPDATE addons INNER JOIN tmp
ON addons.id = tmp.addon_id
SET weeklydownloads = tmp.count""")
transaction.commit_unless_managed()
def _remove_items(self, source_col_name, target_col_name, *objs):
# source_col_name: the PK colname in join_table for the source object
# target_col_name: the PK colname in join_table for the target object
# *objs - objects to remove
# If there aren't any objects, there is nothing to do.
if objs:
# Check that all the objects are of the right type
old_ids = set()
for obj in objs:
if isinstance(obj, self.model):
old_ids.add(obj._get_pk_val())
else:
old_ids.add(obj)
# Remove the specified objects from the join table
cursor = connection.cursor()
cursor.execute("DELETE FROM %s WHERE %s = %%s AND %s IN (%s)" % \
(self.join_table, source_col_name,
target_col_name, ",".join(['%s'] * len(old_ids))),
[self._pk_val] + list(old_ids))
if self.reverse:
sender = self.model
else:
sender = self.instance.__class__
signals.m2m_changed.send(sender=sender, action="remove",
instance=self.instance, model=self.model,
reverse=self.reverse, field_name=self.field_name,
objects=list(objs))
transaction.commit_unless_managed()
def set_no_similarity(request, osm, osm_from, product_id_to, product_id_from):
response = {}
if request.method == 'POST':
nosim = list(NoProductSimilarity.objects.raw("SELECT * FROM matcher_noproductsimilarity WHERE "+osm+"_product_id = %s and "+osm_from+"_product_id = %s", (product_id_to, product_id_from)))
if len(nosim):
# Relation of no similarity already exists, do not do anything
response['status'] = 200
response['msg'] = 'Done'
else:
# The similarity does not exist, create it
# sql_insert = # INSERT QUERY
cursor = connection.cursor()
cursor.execute("INSERT INTO matcher_noproductsimilarity ("+osm+"_product_id, "+osm_from+"_product_id) VALUES ( %s , %s )", [product_id_to, product_id_from])
transaction.commit_unless_managed()
response['status'] = 200
response['msg'] = 'Added to database'
elif request.method == 'DELETE':
nosim = list(NoProductSimilarity.objects.raw("SELECT * FROM matcher_noproductsimilarity WHERE "+osm+"_product_id = %s and "+osm_from+"_product_id = %s", (product_id_to, product_id_from)))
if len(nosim)>0:
nosim = nosim[0]
nosim.delete()
response['status'] = 200
response['msg'] = 'Deleted'
else:
response['status'] = 200
response['msg'] = 'Nothing to do'
else:
response['status'] = 403
response['msg'] = 'Method not handled'
return HttpResponse(json.dumps(response))
def notification_remove_cc(request, case_ids, cc_list):
'''
Description: Remove email addresses from the notification CC list of specific TestCases
Params: $case_ids - Integer/Array: one or more TestCase IDs
$cc_list - Array: contians the email addresses that will
be removed from each TestCase indicated by case_ids.
Returns: JSON. When succeed, status is 0, and message maybe empty or
anything else that depends on the implementation. If something
wrong, status will be 1 and message will be a short description
to the error.
'''
try:
validate_cc_list(cc_list)
except (TypeError, ValidationError):
raise
try:
tc_ids = pre_process_ids(case_ids)
cursor = connection.writer_cursor
ids_values = ",".join(itertools.repeat('%s', len(tc_ids)))
email_values = ",".join(itertools.repeat('%s', len(cc_list)))
sql = TC_REMOVE_CC % (ids_values, email_values)
tc_ids.extend(cc_list)
cursor.execute(sql, tc_ids)
transaction.commit_unless_managed()
except (TypeError, ValueError, Exception):
raise
def migrate_media_field_backwards(model, image_field):
cursor = connection.cursor()
cursor.execute(
"UPDATE %s SET %s ='media/' || %s WHERE %s != ''" %
(model._meta.db_table, image_field, image_field, image_field)
)
transaction.commit_unless_managed()
def handle(self, **options):
'''The main entry point for the Django management command.'''
import_start = datetime.datetime.now()
# Start transaction management.
transaction.commit_unless_managed()
transaction.enter_transaction_management()
transaction.managed(True)
try:
self._promote_devel()
except:
self._rollback_db()
raise
# Finalize the transaction and close the db connection.
transaction.commit()
transaction.leave_transaction_management()
connection.close()
import_end = datetime.datetime.now()
# Print a short summary of what we did.
td = import_end - import_start
print '\nProcessing complete in %s days, %s.%s seconds.' % (
td.days, td.seconds, td.microseconds)
print ' TraitData objects promoted: %s' % (
PublicTraitData.objects.all().count(),)
def get_descendants_group_count(cls, parent=None):
"""
Helper for a very common case: get a group of siblings and the number
of *descendants* in every sibling.
"""
#~
# disclaimer: this is the FOURTH implementation I wrote for this
# function. I really tried to make it return a queryset, but doing so
# with a *single* query isn't trivial with Django's ORM.
# ok, I DID manage to make Django's ORM return a queryset here,
# defining two querysets, passing one subquery in the tables parameters
# of .extra() of the second queryset, using the undocumented order_by
# feature, and using a HORRIBLE hack to avoid django quoting the
# subquery as a table, BUT (and there is always a but) the hack didn't
# survive turning the QuerySet into a ValuesQuerySet, so I just used
# good old SQL.
# NOTE: in case there is interest, the hack to avoid django quoting the
# subquery as a table, was adding the subquery to the alias cache of
# the queryset's query object:
#
# qset.query.quote_cache[subquery] = subquery
#
# If there is a better way to do this in an UNMODIFIED django 1.0, let
# me know.
#~
if parent:
depth = parent.depth + 1
params = cls._get_children_path_interval(parent.path)
extrand = 'AND path BETWEEN %s AND %s'
else:
depth = 1
params = []
extrand = ''
sql = 'SELECT * FROM %(table)s AS t1 INNER JOIN '\
' (SELECT '\
' SUBSTR(path, 1, %(subpathlen)s) AS subpath, '\
' COUNT(1)-1 AS count '\
' FROM %(table)s '\
' WHERE depth >= %(depth)s %(extrand)s'\
' GROUP BY subpath) AS t2 '\
' ON t1.path=t2.subpath '\
' ORDER BY t1.path' % {
'table': connection.ops.quote_name(cls.tb_table_name),
'subpathlen': depth * cls.steplen,
'depth': depth,
'extrand': extrand}
cursor = cls._get_database_cursor('write')
cursor.execute(sql, params)
ret = []
field_names = [field[0] for field in cursor.description]
for node_data in cursor.fetchall():
node = cls(**dict(zip(field_names, node_data[:-2])))
node.descendants_count = node_data[-1]
ret.append(node)
transaction.commit_unless_managed()
return ret
def collection_subscribers():
"""
Collection weekly and monthly subscriber counts.
"""
log.debug('Starting collection subscriber update...')
cursor = connection.cursor()
cursor.execute("""
UPDATE collections SET weekly_subscribers = 0, monthly_subscribers = 0
""")
cursor.execute("""
UPDATE collections AS c
INNER JOIN (
SELECT
COUNT(collection_id) AS count,
collection_id
FROM collection_subscriptions
WHERE created >= DATE_SUB(CURDATE(), INTERVAL 7 DAY)
GROUP BY collection_id
) AS weekly ON (c.id = weekly.collection_id)
INNER JOIN (
SELECT
COUNT(collection_id) AS count,
collection_id
FROM collection_subscriptions
WHERE created >= DATE_SUB(CURDATE(), INTERVAL 31 DAY)
GROUP BY collection_id
) AS monthly ON (c.id = monthly.collection_id)
SET c.weekly_subscribers = weekly.count,
c.monthly_subscribers = monthly.count
""")
transaction.commit_unless_managed()
def update(self, **kwargs):
"""
Updates all elements in the current QuerySet, setting all the given
fields to the appropriate values.
"""
assert self.query.can_filter(), \
"Cannot update a query once a slice has been taken."
query = self.query.clone(sql.UpdateQuery)
query.add_update_values(kwargs)
if not transaction.is_managed():
transaction.enter_transaction_management()
forced_managed = True
else:
forced_managed = False
try:
rows = query.execute_sql(None)
if forced_managed:
transaction.commit()
else:
transaction.commit_unless_managed()
finally:
if forced_managed:
transaction.leave_transaction_management()
self._result_cache = None
return rows
def clean_sessions():
"""
Queue deleting expired session items without breaking poor MySQL
"""
now = timezone.now()
logger = clean_sessions.get_logger()
chunk_size = constance.config.SESSION_CLEANUP_CHUNK_SIZE
if memcache.add(LOCK_ID, now.strftime('%c'), LOCK_EXPIRE):
total_count = get_expired_sessions(now).count()
delete_count = 0
logger.info('Deleting the %s of %s oldest expired sessions' %
(chunk_size, total_count))
try:
cursor = connection.cursor()
delete_count = cursor.execute("""
DELETE
FROM django_session
WHERE expire_date < NOW()
ORDER BY expire_date ASC
LIMIT %s;
""", [chunk_size])
transaction.commit_unless_managed()
finally:
logger.info('Deleted %s expired sessions' % delete_count)
memcache.delete(LOCK_ID)
expired_sessions = get_expired_sessions(now)
if expired_sessions.exists():
clean_sessions.apply_async()
else:
logger.error('The clean_sessions task is already running since %s' %
memcache.get(LOCK_ID))
def move_node(self, node, target, position='last-child'):
"""
Moves ``node`` relative to a given ``target`` node as specified
by ``position`` (when appropriate), by examining both nodes and
calling the appropriate method to perform the move.
A ``target`` of ``None`` indicates that ``node`` should be
turned into a root node.
Valid values for ``position`` are ``'first-child'``,
``'last-child'``, ``'left'`` or ``'right'``.
``node`` will be modified to reflect its new tree state in the
database.
This method explicitly checks for ``node`` being made a sibling
of a root node, as this is a special case due to our use of tree
ids to order root nodes.
"""
if target is None:
if node.is_child_node():
self._make_child_root_node(node)
elif target.is_root_node() and position in ['left', 'right']:
self._make_sibling_of_root_node(node, target, position)
else:
if node.is_root_node():
self._move_root_node(node, target, position)
else:
self._move_child_node(node, target, position)
transaction.commit_unless_managed()
def _base_set(self, mode, key, value, timeout=None):
if timeout is None:
timeout = self.default_timeout
cursor = connection.cursor()
cursor.execute("SELECT COUNT(*) FROM %s" % self._table)
num = cursor.fetchone()[0]
now = datetime.now().replace(microsecond=0)
exp = datetime.fromtimestamp(time.time() + timeout).replace(microsecond=0)
if num > self._max_entries:
self._cull(cursor, now)
encoded = base64.encodestring(pickle.dumps(value, 2)).strip()
cursor.execute("SELECT cache_key, expires FROM %s WHERE cache_key = %%s" % self._table, [key])
try:
result = cursor.fetchone()
if result and (mode == 'set' or
(mode == 'add' and result[1] < now)):
cursor.execute("UPDATE %s SET value = %%s, expires = %%s WHERE cache_key = %%s" % self._table, [encoded, str(exp), key])
else:
cursor.execute("INSERT INTO %s (cache_key, value, expires) VALUES (%%s, %%s, %%s)" % self._table, [key, encoded, str(exp)])
except DatabaseError:
# To be threadsafe, updates/inserts are allowed to fail silently
return False
else:
transaction.commit_unless_managed()
return True
def _add_items(self, source_col_name, target_col_name, *objs):
# join_table: name of the m2m link table
# source_col_name: the PK colname in join_table for the source object
# target_col_name: the PK colname in join_table for the target object
# *objs - objects to add. Either object instances, or primary keys of object instances.
# If there aren't any objects, there is nothing to do.
if objs:
from django.db.models.base import Model
# Check that all the objects are of the right type
new_ids = set()
for obj in objs:
if isinstance(obj, self.model):
new_ids.add(obj._get_pk_val())
elif isinstance(obj, Model):
raise TypeError, "'%s' instance expected" % self.model._meta.object_name
else:
new_ids.add(obj)
# Add the newly created or already existing objects to the join table.
# First find out which items are already added, to avoid adding them twice
cursor = connection.cursor()
cursor.execute("SELECT %s FROM %s WHERE %s = %%s AND %s IN (%s)" % \
(target_col_name, self.join_table, source_col_name,
target_col_name, ",".join(['%s'] * len(new_ids))),
[self._pk_val] + list(new_ids))
existing_ids = set([row[0] for row in cursor.fetchall()])
# Add the ones that aren't there already
for obj_id in (new_ids - existing_ids):
cursor.execute("INSERT INTO %s (%s, %s) VALUES (%%s, %%s)" % \
(self.join_table, source_col_name, target_col_name),
[self._pk_val, obj_id])
transaction.commit_unless_managed()
def handle_app(self, app, **options):
using = options.get('database', DEFAULT_DB_ALIAS)
connection = connections[using]
app_name = app.__name__.split('.')[-2]
self.style = no_style()
sql_list = sql_reset(app, self.style, connection)
if options.get('interactive'):
confirm = raw_input("""
You have requested a database reset.
This will IRREVERSIBLY DESTROY any data for
the "%s" application in the database "%s".
Are you sure you want to do this?
Type 'yes' to continue, or 'no' to cancel: """ %
(app_name, connection.settings_dict['NAME']))
else:
confirm = 'yes'
if confirm == 'yes':
try:
cursor = connection.cursor()
for sql in sql_list:
cursor.execute(sql)
except Exception, e:
transaction.rollback_unless_managed()
raise CommandError(
"""Error: %s couldn't be reset. Possible reasons:
* The database isn't running or isn't configured correctly.
* At least one of the database tables doesn't exist.
* The SQL was invalid.
Hint: Look at the output of 'django-admin.py sqlreset %s'. That's the SQL this command wasn't able to run.
The full error: %s""" % (app_name, app_name, e))
transaction.commit_unless_managed()
def set_default_screenshots():
cur = connection.cursor()
cur2 = connection.cursor()
# unset default_screenshot for productions that don't have any processed screenshots
# (if things are working properly then we would expect this to be redundant. But no harm in being thorough :-) )
cur.execute('''
UPDATE demoscene_production SET default_screenshot_id = NULL
WHERE id NOT IN (SELECT DISTINCT production_id FROM demoscene_screenshot WHERE thumbnail_url <> '')
''')
# get all processed screenshots (= ones that have thumbnails), group by production_id
cur.execute(
"SELECT id, production_id FROM demoscene_screenshot WHERE thumbnail_url <> '' ORDER BY production_id"
)
for production_id, screenshots in groupby(cur, lambda row: row[1]):
# select a random screenshot from the group and set it as default_screenshot for that prod
screenshot_id = random.choice([row[0] for row in screenshots])
cur2.execute(
'''
UPDATE demoscene_production SET default_screenshot_id = %s WHERE id = %s
''', [screenshot_id, production_id])
transaction.commit_unless_managed()
def weekly_downloads():
"""
Update 7-day add-on download counts.
"""
raise_if_reindex_in_progress()
cursor = connection.cursor()
cursor.execute("""
SELECT addon_id, SUM(count) AS weekly_count
FROM download_counts
WHERE `date` >= DATE_SUB(CURDATE(), INTERVAL 7 DAY)
GROUP BY addon_id
ORDER BY addon_id""")
counts = cursor.fetchall()
addon_ids = [r[0] for r in counts]
if not addon_ids:
return
cursor.execute(
"""
SELECT id, 0
FROM addons
WHERE id NOT IN %s""", (addon_ids, ))
counts += cursor.fetchall()
cursor.execute("""
CREATE TEMPORARY TABLE tmp_wd
(addon_id INT PRIMARY KEY, count INT)""")
cursor.execute(
'INSERT INTO tmp_wd VALUES %s' % ','.join(['(%s,%s)'] * len(counts)),
list(itertools.chain(*counts)))
cursor.execute("""
UPDATE addons INNER JOIN tmp_wd
ON addons.id = tmp_wd.addon_id
SET weeklydownloads = tmp_wd.count""")
cursor.execute("DROP TABLE IF EXISTS tmp_wd")
transaction.commit_unless_managed()
def load_bulk(cls, bulk_data, parent=None, keep_ids=False):
"""Loads a list/dictionary structure to the tree."""
cls = get_result_class(cls)
# tree, iterative preorder
added = []
if parent:
parent_id = parent.pk
else:
parent_id = None
# stack of nodes to analize
stack = [(parent_id, node) for node in bulk_data[::-1]]
foreign_keys = cls.get_foreign_keys()
while stack:
parent_id, node_struct = stack.pop()
# shallow copy of the data strucure so it doesn't persist...
node_data = node_struct['data'].copy()
cls._process_foreign_keys(foreign_keys, node_data)
if keep_ids:
node_data['id'] = node_struct['id']
if parent_id:
parent = cls.objects.get(pk=parent_id)
node_obj = parent.add_child(**node_data)
else:
node_obj = cls.add_root(**node_data)
added.append(node_obj.pk)
if 'children' in node_struct:
# extending the stack with the current node as the parent of
# the new nodes
stack.extend([
(node_obj.pk, node)
for node in node_struct['children'][::-1]
])
transaction.commit_unless_managed()
return added
def sync_db(self):
'''
This function uses the same code that's used in syncdb to dynamically
execute DDL sql on-the-fly. Copied from:
/usr/local/lib/python2.6/dist-packages/django/core/management/commands/syncdb.py
'''
from django.core.management.color import no_style
from django.db import connection, transaction
cursor = connection.cursor()
tables = connection.introspection.table_names()
seen_models = connection.introspection.installed_models(tables)
created_models = set()
pending_references = {}
sql, references = connection.creation.sql_create_model(
self.model_class, no_style(), seen_models)
seen_models.add(self.model_class)
created_models.add(self.model_class)
for refto, refs in references.items():
pending_references.setdefault(refto, []).extend(refs)
if refto in seen_models:
sql.extend(
connection.creation.sql_for_pending_references(
refto, no_style(), pending_references))
sql.extend(
connection.creation.sql_for_pending_references(
self.model_class, no_style(), pending_references))
#append point geometry by calling the PostGIS function:
sql.append(
"select AddGeometryColumn('public','%s','point',4326,'POINT',2)" %
self.form.table_name)
for statement in sql:
cursor.execute(statement)
transaction.commit_unless_managed()
def move_node(self, node, target, position='last-child'):
"""
Moves ``node`` relative to a given ``target`` node as specified
by ``position`` (when appropriate), by examining both nodes and
calling the appropriate method to perform the move.
A ``target`` of ``None`` indicates that ``node`` should be
turned into a root node.
Valid values for ``position`` are ``'first-child'``,
``'last-child'``, ``'left'`` or ``'right'``.
``node`` will be modified to reflect its new tree state in the
database.
This method explicitly checks for ``node`` being made a sibling
of a root node, as this is a special case due to our use of tree
ids to order root nodes.
"""
if self._base_manager:
return self._base_manager.move_node(node,
target,
position=position)
if target is None:
if node.is_child_node():
self._make_child_root_node(node)
elif target.is_root_node() and position in ['left', 'right']:
self._make_sibling_of_root_node(node, target, position)
else:
if node.is_root_node():
self._move_root_node(node, target, position)
else:
self._move_child_node(node, target, position)
transaction.commit_unless_managed()
def add_root(cls, **kwargs):
"""Adds a root node to the tree."""
# do we have a root node already?
last_root = cls.get_last_root_node()
if last_root and last_root.node_order_by:
# there are root nodes and node_order_by has been set
# delegate sorted insertion to add_sibling
return last_root.add_sibling('sorted-sibling', **kwargs)
if last_root:
# adding the new root node as the last one
newtree_id = last_root.tree_id + 1
else:
# adding the first root node
newtree_id = 1
if len(kwargs) == 1 and 'instance' in kwargs:
# adding the passed (unsaved) instance to the tree
newobj = kwargs['instance']
if newobj.pk:
raise NodeAlreadySaved("Attempted to add a tree node that is "\
"already in the database")
else:
# creating the new object
newobj = get_result_class(cls)(**kwargs)
newobj.depth = 1
newobj.tree_id = newtree_id
newobj.lft = 1
newobj.rgt = 2
# saving the instance before returning it
newobj.save()
transaction.commit_unless_managed()
return newobj
def add_child(self, **kwargs):
"Adds a child to the node."
if not self.is_leaf():
# there are child nodes, delegate insertion to add_sibling
if self.node_order_by:
pos = 'sorted-sibling'
else:
pos = 'last-sibling'
last_child = self.get_last_child()
last_child._cached_parent_obj = self
return last_child.add_sibling(pos, **kwargs)
# we're adding the first child of this node
sql, params = self.__class__._move_right(self.tree_id, self.rgt, False,
2)
# creating a new object
newobj = self.__class__(**kwargs)
newobj.tree_id = self.tree_id
newobj.depth = self.depth + 1
newobj.lft = self.lft + 1
newobj.rgt = self.lft + 2
# this is just to update the cache
self.rgt = self.rgt + 2
newobj._cached_parent_obj = self
cursor = connection.cursor()
cursor.execute(sql, params)
# saving the instance before returning it
newobj.save()
transaction.commit_unless_managed()
return newobj
def bulk_insert_ignore(self,fields,value_list,print_sql=False):
"""主要解决mysql重复数据过滤的问题 加入了 ignore 关键词"""
db_table = self.model._meta.db_table
values_sql="(%s)" %(','.join([" %s " for _ in range(len(fields))]))
base_sql = "INSERT IGNORE INTO %s (%s) VALUES " % (db_table, ",".join(fields))
sql = """%s %s""" % (base_sql, values_sql)
from django.db import connection,transaction
cursor = connection.cursor()
try:
f=cursor.executemany(sql, value_list)
k=transaction.commit_unless_managed()
import pdb;pdb.set_trace()
return True
except Exception as e:
print e
return False
class DatabaseCache(BaseDatabaseCache):
# This class uses cursors provided by the database connection. This means
# it reads expiration values as aware or naive datetimes depending on the
# value of USE_TZ. They must be compared to aware or naive representations
# of "now" respectively.
# But it bypasses the ORM for write operations. As a consequence, aware
# datetimes aren't made naive for databases that don't support time zones. 市区
# We work around this problem by always using naive datetimes when writing
# expiration values 过期值, in UTC when USE_TZ = True and in local time otherwise.
def get(self, key, default=None, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
self.cache_model_class 不懂
db = router.db_for_read(self.cache_model_class)
table = connections[db].ops.quote_name(self._table)
cursor = connections[db].cursor() 游标
cursor.execute("SELECT cache_key, value, expires FROM %s "
"WHERE cache_key = %%s" % table, [key]) table
row = cursor.fetchone() 取一个
if row is None: 空集
return default
now = timezone.now()
row[2] 是时间
if row[2] < now: 事件比较新
db = router.db_for_write(self.cache_model_class)
cursor = connections[db].cursor()
cursor.execute("DELETE FROM %s "
"WHERE cache_key = %%s" % table, [key])
transaction.commit_unless_managed(using=db) 事务
return default
class Migration(SchemaMigration):
def forwards(self, orm):
cursor = connection.cursor()
for model in ('urltoken', 'offsiteinvite'):
try:
cursor.execute("SELECT * FROM registration_%s LIMIT 1" % model)
db.rename_table('registration_%s' % model,
'jmbo_registration_%s' % model)
if orm['contenttypes.ContentType'].objects.filter(
app_label='registration', model=model).exists():
orm['contenttypes.ContentType'].objects.filter(
app_label='jmbo_registration', model=model).delete()
orm['contenttypes.ContentType'].objects.filter(
app_label='registration',
model=model).update(app_label='jmbo_registration')
except DatabaseError, e:
if 'does not exist' not in e.message:
raise e
transaction.commit_unless_managed()
class Command(BaseCommand):
"""
Example: python manage.py drop corporate_memberships memberships
"""
def run_sql(self, sql, **options):
"""
Parse through sql statements.
Execute each sql statement.
Commit in one transaction.
"""
# sql statement regex
statements = re.compile(r";[ \t]*$", re.M)
sql_list = []
# database information
using = options.get('database', DEFAULT_DB_ALIAS)
connection = connections[using]
for statement in statements.split(sql.decode(settings.FILE_CHARSET)):
# Remove any comments from the file
statement = re.sub(ur"--.*([\n\Z]|$)", "", statement)
if statement.strip():
sql_list.append(statement + u";")
try:
cursor = connection.cursor()
cursor.execute('SET FOREIGN_KEY_CHECKS = 0;')
for sql in sql_list:
cursor.execute(sql)
cursor.execute('SET FOREIGN_KEY_CHECKS = 1;')
except Exception, e:
transaction.rollback_unless_managed()
raise CommandError("""
Error running SQL. Possible reasons:
* The database isn't running or isn't configured correctly.
* At least one of the database tables doesn't exist.
* The SQL was invalid.
The full error: %s""" % (e))
transaction.commit_unless_managed()
return sql_list
def delete_mp(campaign_id):
ad = Ad.objects.get(DE_campaign_id=campaign_id)
from django.db import connection, transaction
cursor = connection.cursor()
cursor.execute('delete from media_planning_de_clickdata where ad_id=%s',
[ad.id])
transaction.commit_unless_managed()
cursor.execute('delete from media_planning_flight where ad_id=%s', [ad.id])
transaction.commit_unless_managed()
cursor.execute('delete from ad_adrelatedstaff where ad_id=%s', [ad.id])
transaction.commit_unless_managed()
ad.delete()
return True
def actualizar_secuencias():
# select setval('main_familia_id_seq', (select max(id)+1 from main_familia), false)
cursor = connection.cursor()
cursor.execute(
"select setval('main_centrofamiliar_id_seq', (select max(id)+1 from main_centrofamiliar), false)"
)
transaction.commit_unless_managed()
cursor.execute(
"select setval('main_familia_id_seq', (select max(id)+1 from main_familia), false)"
)
transaction.commit_unless_managed()
cursor.execute(
"select setval('main_persona_id_seq', (select max(id)+1 from main_persona), false)"
)
transaction.commit_unless_managed()
def clear_trashinfo():
from django.db import connection, transaction
cursor = connection.cursor()
cursor.execute(
'delete from media_planning_de_clickdata where ad_id not in (select id from ad_ad)'
)
transaction.commit_unless_managed()
cursor.execute(
'delete from media_planning_flight where ad_id not in (select id from ad_ad)'
)
transaction.commit_unless_managed()
cursor.execute(
'delete from ad_adrelatedstaff where ad_id not in (select id from ad_ad)'
)
transaction.commit_unless_managed()
return True
def merge_channel(media_id):
# For same media, merge the channels owning the same name
# Remove spaces in channels' name # Find same name channels, store them into dict. {channel_name:[channel_id_1, channel_id_2, ...]}
try:
media = Media.objects.get(id=media_id)
except ObjectDoesNotExist:
print 'The media dose not exist!'
return False
channel_name_dict = {}
for channel in media.related_channels.all():
channel.c_name = channel.c_name.replace(' ', '')
channel.save()
if channel_name_dict.has_key(channel.c_name):
channel_name_dict[channel.c_name].append(channel.id)
else:
channel_name_dict[channel.c_name] = [channel.id]
# Then, update related tables
from django.db import connection, transaction
cursor = connection.cursor()
for val in channel_name_dict.values():
if len(val) > 1:
val.sort()
for i in val[1:]:
cursor.execute(
'update media_mediaadinfo set channel_id=%s where channel_id=%s',
[val[0], i])
transaction.commit_unless_managed()
cursor.execute(
'update media_planning_flight set channel_id=%s where channel_id=%s',
[val[0], i])
transaction.commit_unless_managed()
cursor.execute(
'update media_planning_de_clickdata set channel_id=%s where channel_id=%s',
[val[0], i])
transaction.commit_unless_managed()
channel = Channel.objects.get(id=i)
channel.delete()
# At last, call merge_flight channels needed
for channel in media.related_channels.all():
merge_flight(channel)
def new_topic(poll):
subject = "Poll: %s" % poll.title
descr = poll.description
game = poll.game
now = int(mktime(gmtime()))
cursor = connections['fluxbb'].cursor()
#cursor.execute("SELECT id FROM users WHERE username = '******'", ['longturn'])
cursor.execute("SELECT id FROM users WHERE username = '******'")
poster_id = cursor.fetchone()[0]
cursor.execute("SELECT id FROM forums WHERE forum_name = '%s'",
[AsIs(game)])
forum_id = cursor.fetchone()[0]
cmd = "INSERT INTO topics (poster, subject, forum_id, last_poster, posted, last_post) VALUES ('%s', '%s', %d, '%s', %d, %d)"
cursor.execute(
cmd,
['longturn', AsIs(subject), forum_id, 'longturn', now, now])
transaction.commit_unless_managed(using='fluxbb')
cursor.execute(
"SELECT max(id) FROM topics WHERE poster = '%s' AND subject = '%s'",
['longturn', AsIs(subject)])
topic_id = cursor.fetchone()[0]
cmd = "INSERT INTO posts (poster, poster_id, topic_id, posted, message) VALUES ('%s', %d, %d, %d, '%s')"
cursor.execute(cmd, ['longturn', poster_id, topic_id, now, AsIs(descr)])
transaction.commit_unless_managed(using='fluxbb')
cmd = "UPDATE forums SET num_topics = num_topics + 1, num_posts = num_posts + 1, last_post = %d, last_poster = '%s' WHERE forum_name = '%s'"
cursor.execute(cmd, [now, 'longturn', AsIs(game)])
transaction.commit_unless_managed(using='fluxbb')
return topic_id
def fix_tree(cls, destructive=False):
"""
Solves some problems that can appear when transactions are not used and
a piece of code breaks, leaving the tree in an inconsistent state.
The problems this method solves are:
1. Nodes with an incorrect ``depth`` or ``numchild`` values due to
incorrect code and lack of database transactions.
2. "Holes" in the tree. This is normal if you move/delete nodes a
lot. Holes in a tree don't affect performance,
3. Incorrect ordering of nodes when ``node_order_by`` is enabled.
Ordering is enforced on *node insertion*, so if an attribute in
``node_order_by`` is modified after the node is inserted, the
tree ordering will be inconsistent.
:param destructive:
A boolean value. If True, a more agressive fix_tree method will be
attemped. If False (the default), it will use a safe (and fast!)
fix approach, but it will only solve the ``depth`` and
``numchild`` nodes, it won't fix the tree holes or broken path
ordering.
.. warning::
Currently what the ``destructive`` method does is:
1. Backup the tree with :meth:`dump_data`
2. Remove all nodes in the tree.
3. Restore the tree with :meth:`load_data`
So, even when the primary keys of your nodes will be preserved,
this method isn't foreign-key friendly. That needs complex
in-place tree reordering, not available at the moment (hint:
patches are welcome).
"""
if destructive:
dump = cls.dump_bulk(None, True)
cls.objects.all().delete()
cls.load_bulk(dump, None, True)
else:
cursor = cls._get_database_cursor('write')
# fix the depth field
# we need the WHERE to speed up postgres
sql = "UPDATE %s "\
"SET depth=LENGTH(path)/%%s "\
"WHERE depth!=LENGTH(path)/%%s" % (
connection.ops.quote_name(cls.tb_table_name), )
vals = [cls.steplen, cls.steplen]
cursor.execute(sql, vals)
# fix the numchild field
vals = ['_' * cls.steplen]
# the cake and sql portability are a lie
if cls.get_database_vendor('read') == 'mysql':
sql = "SELECT tbn1.path, tbn1.numchild, ("\
"SELECT COUNT(1) "\
"FROM %(table)s AS tbn2 "\
"WHERE tbn2.path LIKE "\
"CONCAT(tbn1.path, %%s)) AS real_numchild "\
"FROM %(table)s AS tbn1 "\
"HAVING tbn1.numchild != real_numchild" % {
'table': connection.ops.quote_name(
cls.tb_table_name)}
else:
subquery = "(SELECT COUNT(1) FROM %(table)s AS tbn2"\
" WHERE tbn2.path LIKE tbn1.path||%%s)"
sql = ("SELECT tbn1.path, tbn1.numchild, " + subquery +
" FROM %(table)s AS tbn1 WHERE tbn1.numchild != " +
subquery)
sql = sql % {
'table': connection.ops.quote_name(cls.tb_table_name)}
# we include the subquery twice
vals *= 2
cursor.execute(sql, vals)
sql = "UPDATE %(table)s "\
"SET numchild=%%s "\
"WHERE path=%%s" % {
'table': connection.ops.quote_name(cls.tb_table_name)}
for node_data in cursor.fetchall():
vals = [node_data[2], node_data[0]]
cursor.execute(sql, vals)
transaction.commit_unless_managed()
def SetupQueueThread(
tid,
run,
probe_servers,
progress_queue,
):
connection.close()
try:
while True:
item = probe_servers.get()
if isinstance(item, int):
try:
item = Probe.Server.objects.get(id=item)
except:
probe_servers.task_done()
continue
hostname = item.servername.strip()
if (not hostname or any([
x in hostname for x in " \t%/&#\"'\\{[]}()*,;<>$"
]) or any([ord(x) >= 128 or ord(x) <= 32 for x in hostname])):
item.enabled = False
item.save()
probe_servers.task_done()
continue
hostname = hostname.strip(".")
while hostname.find("..") >= 0:
hostname = hostname.replace("..", ".")
if hostname != item.servername:
item.enabled = False
item.save()
item = "0," + hostname + "," + str(
item.port
) + "," + item.protocol # Convert to string to correct the list
if not isinstance(item, Probe.Server):
hostname_line = item
if not hostname_line.strip():
probe_servers.task_done()
continue
split_line = hostname_line.strip().split(",")
if len(split_line) > 3:
(index, hostname, port, protocol) = split_line[:4]
port = port.strip()
protocol = protocol.strip()
if not protocol or protocol.upper(
) not in Probe.Server.PROTOCOL_PORT_MAP:
protocol = Probe.Server.PROTOCOL_PORT.get(
int(port) if port else 443,
Probe.Server.PROTOCOL_HTTPS)
elif len(split_line) > 2:
(index, hostname, port) = split_line[:3]
port = port.strip()
protocol = Probe.Server.PROTOCOL_PORT.get(
int(port) if port else 443,
Probe.Server.PROTOCOL_HTTPS)
else:
port = ""
(var1, var2) = split_line
if var1.isdigit():
(index, hostname) = (var1, var2)
else:
(hostname, port) = (var1, var2)
port = port.strip()
protocol = Probe.Server.PROTOCOL_PORT.get(
int(port) if port else 443,
Probe.Server.PROTOCOL_HTTPS)
hostname = hostname.strip()
if (not hostname or any([
x in hostname for x in " \t%/&#\"'\\{[]}()*,;<>$"
]) or any([ord(x) >= 128 or ord(x) <= 32 for x in hostname])):
probe_servers.task_done()
continue
hostname = hostname.strip(".")
while hostname.find("..") >= 0:
hostname = hostname.replace("..", ".")
if not port:
port = 443
else:
port = int(port)
sn_t = "%s:%05d:%s" % (hostname, port, protocol)
(item, created) = Probe.Server.objects.get_or_create(
full_servername=sn_t,
defaults={
'enabled': True,
"alexa_rating": 0,
"servername": hostname,
"port": port,
"protocol": protocol.upper(),
})
if created:
item.Construct()
if item.enabled:
try:
#run_entry = Probe.ProbeQueue.objects.create(part_of_run=run,server=item,state=Probe.ProbeQueue.PROBEQ_IDLE)
sid = transaction.savepoint()
cursor = connection.cursor()
cursor.execute(
"""INSERT INTO probedata2_probequeue (part_of_run_id, server_id, state)
VALUES (%s,%s,%s)""", [
str(run.id),
str(item.id), Probe.ProbeQueue.PROBEQ_IDLE
])
transaction.savepoint_commit(sid)
transaction.commit_unless_managed()
progress_queue.put(True)
except:
transaction.savepoint_rollback(sid)
pass
probe_servers.task_done()
except:
pass
def save_base(self,
raw=False,
cls=None,
origin=None,
force_insert=False,
force_update=False,
using=None):
"""
Does the heavy-lifting involved in saving. Subclasses shouldn't need to
override this method. It's separate from save() in order to hide the
need for overrides of save() to pass around internal-only parameters
('raw', 'cls', and 'origin').
"""
using = using or router.db_for_write(self.__class__, instance=self)
assert not (force_insert and force_update)
if cls is None:
cls = self.__class__
meta = cls._meta
if not meta.proxy:
origin = cls
else:
meta = cls._meta
if origin and not meta.auto_created:
signals.pre_save.send(sender=origin,
instance=self,
raw=raw,
using=using)
# If we are in a raw save, save the object exactly as presented.
# That means that we don't try to be smart about saving attributes
# that might have come from the parent class - we just save the
# attributes we have been given to the class we have been given.
# We also go through this process to defer the save of proxy objects
# to their actual underlying model.
if not raw or meta.proxy:
if meta.proxy:
org = cls
else:
org = None
for parent, field in meta.parents.items():
# At this point, parent's primary key field may be unknown
# (for example, from administration form which doesn't fill
# this field). If so, fill it.
if field and getattr(
self, parent._meta.pk.attname) is None and getattr(
self, field.attname) is not None:
setattr(self, parent._meta.pk.attname,
getattr(self, field.attname))
self.save_base(cls=parent, origin=org, using=using)
if field:
setattr(self, field.attname,
self._get_pk_val(parent._meta))
if meta.proxy:
return
if not meta.proxy:
non_pks = [f for f in meta.local_fields if not f.primary_key]
# First, try an UPDATE. If that doesn't update anything, do an INSERT.
pk_val = self._get_pk_val(meta)
pk_set = pk_val is not None
record_exists = True
manager = cls._base_manager
if pk_set:
# Determine whether a record with the primary key already exists.
if (force_update or
(not force_insert
and manager.using(using).filter(pk=pk_val).exists())):
# It does already exist, so do an UPDATE.
if force_update or non_pks:
values = [(f, None, (raw and getattr(self, f.attname)
or f.pre_save(self, False)))
for f in non_pks]
if values:
rows = manager.using(using).filter(
pk=pk_val)._update(values)
if force_update and not rows:
raise DatabaseError(
"Forced update did not affect any rows.")
else:
record_exists = False
if not pk_set or not record_exists:
if meta.order_with_respect_to:
# If this is a model with an order_with_respect_to
# autopopulate the _order field
field = meta.order_with_respect_to
order_value = manager.using(using).filter(
**{
field.name: getattr(self, field.attname)
}).count()
self._order = order_value
fields = meta.local_fields
if not pk_set:
if force_update:
raise ValueError(
"Cannot force an update in save() with no primary key."
)
fields = [
f for f in fields if not isinstance(f, AutoField)
]
record_exists = False
update_pk = bool(meta.has_auto_field and not pk_set)
result = manager._insert([self],
fields=fields,
return_id=update_pk,
using=using,
raw=raw)
if update_pk:
setattr(self, meta.pk.attname, result)
transaction.commit_unless_managed(using=using)
# Store the database on which the object was saved
self._state.db = using
# Once saved, this is no longer a to-be-added instance.
self._state.adding = False
# Signal that the save is complete
if origin and not meta.auto_created:
signals.post_save.send(sender=origin,
instance=self,
created=(not record_exists),
raw=raw,
using=using)
return
event.group = group
for view in views:
group.views.add(view)
# save the event unless its been sampled
if not is_sample:
try:
event.save()
except IntegrityError:
transaction.rollback_unless_managed(using=group._state.db)
return event
transaction.commit_unless_managed(using=group._state.db)
if settings.USE_SEARCH:
try:
maybe_delay(index_event, event)
except Exception, e:
transaction.rollback_unless_managed(using=group._state.db)
logger.exception(u'Error indexing document: %s', e)
if is_new:
try:
regression_signal.send(sender=self.model, instance=group)
except Exception, e:
transaction.rollback_unless_managed(using=group._state.db)
logger.exception(u'Error sending regression signal: %s', e)
def delete(self, key):
cursor = connection.cursor()
cursor.execute("DELETE FROM %s WHERE cache_key = %%s" % self._table, [key])
transaction.commit_unless_managed()
def handle(self, *fixture_labels, **options):
using = options.get('database', DEFAULT_DB_ALIAS)
connection = connections[using]
self.style = no_style()
verbosity = int(options.get('verbosity', 1))
show_traceback = options.get('traceback', False)
# commit is a stealth option - it isn't really useful as
# a command line option, but it can be useful when invoking
# loaddata from within another script.
# If commit=True, loaddata will use its own transaction;
# if commit=False, the data load SQL will become part of
# the transaction in place when loaddata was invoked.
commit = options.get('commit', True)
# Keep a count of the installed objects and fixtures
fixture_count = 0
loaded_object_count = 0
fixture_object_count = 0
models = set()
humanize = lambda dirname: dirname and "'%s'" % dirname or 'absolute path'
# Get a cursor (even though we don't need one yet). This has
# the side effect of initializing the test database (if
# it isn't already initialized).
cursor = connection.cursor()
# Start transaction management. All fixtures are installed in a
# single transaction to ensure that all references are resolved.
if commit:
transaction.commit_unless_managed(using=using)
transaction.enter_transaction_management(using=using)
transaction.managed(True, using=using)
class SingleZipReader(zipfile.ZipFile):
def __init__(self, *args, **kwargs):
zipfile.ZipFile.__init__(self, *args, **kwargs)
if settings.DEBUG:
assert len(
self.namelist()
) == 1, "Zip-compressed fixtures must contain only one file."
def read(self):
return zipfile.ZipFile.read(self, self.namelist()[0])
compression_types = {
None: file,
'gz': gzip.GzipFile,
'zip': SingleZipReader
}
if has_bz2:
compression_types['bz2'] = bz2.BZ2File
app_module_paths = []
for app in get_apps():
if hasattr(app, '__path__'):
# It's a 'models/' subpackage
for path in app.__path__:
app_module_paths.append(path)
else:
# It's a models.py module
app_module_paths.append(app.__file__)
app_fixtures = [
os.path.join(os.path.dirname(path), 'fixtures')
for path in app_module_paths
]
for fixture_label in fixture_labels:
parts = fixture_label.split('.')
if len(parts) > 1 and parts[-1] in compression_types:
compression_formats = [parts[-1]]
parts = parts[:-1]
else:
compression_formats = compression_types.keys()
if len(parts) == 1:
fixture_name = parts[0]
formats = serializers.get_public_serializer_formats()
else:
fixture_name, format = '.'.join(parts[:-1]), parts[-1]
if format in serializers.get_public_serializer_formats():
formats = [format]
else:
formats = []
if formats:
if verbosity > 1:
self.stdout.write("Loading '%s' fixtures...\n" %
fixture_name)
else:
self.stderr.write(
self.style.ERROR(
"Problem installing fixture '%s': %s is not a known serialization format.\n"
% (fixture_name, format)))
if commit:
transaction.rollback(using=using)
transaction.leave_transaction_management(using=using)
return
if os.path.isabs(fixture_name):
fixture_dirs = [fixture_name]
else:
fixture_dirs = app_fixtures + list(
settings.FIXTURE_DIRS) + ['']
for fixture_dir in fixture_dirs:
if verbosity > 1:
self.stdout.write("Checking %s for fixtures...\n" %
humanize(fixture_dir))
label_found = False
for combo in product([using, None], formats,
compression_formats):
database, format, compression_format = combo
file_name = '.'.join(
p for p in
[fixture_name, database, format, compression_format]
if p)
if verbosity > 1:
self.stdout.write("Trying %s for %s fixture '%s'...\n" % \
(humanize(fixture_dir), file_name, fixture_name))
full_path = os.path.join(fixture_dir, file_name)
open_method = compression_types[compression_format]
try:
fixture = open_method(full_path, 'r')
if label_found:
fixture.close()
self.stderr.write(
self.style.ERROR(
"Multiple fixtures named '%s' in %s. Aborting.\n"
% (fixture_name, humanize(fixture_dir))))
if commit:
transaction.rollback(using=using)
transaction.leave_transaction_management(
using=using)
return
else:
fixture_count += 1
objects_in_fixture = 0
loaded_objects_in_fixture = 0
if verbosity > 0:
self.stdout.write("Installing %s fixture '%s' from %s.\n" % \
(format, fixture_name, humanize(fixture_dir)))
try:
objects = serializers.deserialize(format,
fixture,
using=using)
for obj in objects:
objects_in_fixture += 1
if router.allow_syncdb(
using, obj.object.__class__):
loaded_objects_in_fixture += 1
models.add(obj.object.__class__)
obj.save(using=using)
loaded_object_count += loaded_objects_in_fixture
fixture_object_count += objects_in_fixture
label_found = True
except (SystemExit, KeyboardInterrupt):
raise
except Exception:
import traceback
fixture.close()
if commit:
transaction.rollback(using=using)
transaction.leave_transaction_management(
using=using)
if show_traceback:
traceback.print_exc()
else:
self.stderr.write(
self.style.ERROR(
"Problem installing fixture '%s': %s\n"
% (full_path, ''.join(
traceback.format_exception(
sys.exc_type,
sys.exc_value,
sys.exc_traceback)))))
return
fixture.close()
# If the fixture we loaded contains 0 objects, assume that an
# error was encountered during fixture loading.
if objects_in_fixture == 0:
self.stderr.write(
self.style.ERROR(
"No fixture data found for '%s'. (File format may be invalid.)\n"
% (fixture_name)))
if commit:
transaction.rollback(using=using)
transaction.leave_transaction_management(
using=using)
return
except Exception, e:
if verbosity > 1:
self.stdout.write("No %s fixture '%s' in %s.\n" % \
(format, fixture_name, humanize(fixture_dir)))
def handle(self, *args, **options):
transaction.commit_unless_managed()
transaction.enter_transaction_management()
transaction.managed(True)
site = Site.objects.all()[0]
cursor = connection.cursor()
cursor.execute(FREE_SQL)
for row in cursor:
(content_type_id, object_id, parent_id, name, website, email,
date_submitted, date_modified, date_approved, comment, markup,
is_public, is_approved, ip_address) = row
tc = ThreadedComment(
content_type_id=content_type_id,
object_pk=object_id,
user_name=name,
user_email=email,
user_url=website,
comment=comment,
submit_date=date_submitted,
ip_address=ip_address,
is_public=is_public,
is_removed=not is_approved,
parent_id=parent_id,
site=site,
)
tc.save(skip_tree_path=True)
cursor = connection.cursor()
cursor.execute(USER_SQL)
for row in cursor:
(content_type_id, object_id, parent_id, user_id, date_submitted,
date_modified, date_approved, comment, markup, is_public,
is_approved, ip_address) = row
tc = ThreadedComment(
content_type_id=content_type_id,
object_pk=object_id,
user_id=user_id,
comment=comment,
submit_date=date_submitted,
ip_address=ip_address,
is_public=is_public,
is_removed=not is_approved,
parent_id=parent_id,
site=site,
)
tc.save(skip_tree_path=True)
for comment in ThreadedComment.objects.all():
path = [str(comment.id).zfill(PATH_DIGITS)]
current = comment
while current.parent:
current = current.parent
path.append(str(current.id).zfill(PATH_DIGITS))
comment.tree_path = PATH_SEPARATOR.join(reversed(path))
comment.save(skip_tree_path=True)
if comment.parent:
ThreadedComment.objects.filter(pk=comment.parent.pk).update(
last_child=comment)
transaction.commit()
transaction.leave_transaction_management()
else:
transaction.commit_unless_managed()
else:
if verbosity >= 2:
print "No custom SQL for %s.%s model" % (
app_name, model._meta.object_name)
# Install SQL indicies for all newly created models
for app in models.get_apps():
app_name = app.__name__.split('.')[-2]
for model in models.get_models(app):
if model in created_models:
index_sql = connection.creation.sql_indexes_for_model(
model, self.style)
if index_sql:
if verbosity >= 1:
print "Installing index for %s.%s model" % (
app_name, model._meta.object_name)
try:
for sql in index_sql:
cursor.execute(sql)
except Exception, e:
sys.stderr.write("Failed to install index for %s.%s model: %s\n" % \
(app_name, model._meta.object_name, e))
transaction.rollback_unless_managed()
else:
transaction.commit_unless_managed()
# Install the 'initial_data' fixture, using format discovery
from django.core.management import call_command
call_command('loaddata', 'initial_data', verbosity=verbosity)
def handle_noargs(self, **options):
verbosity = int(options.get('verbosity'))
interactive = options.get('interactive')
show_traceback = options.get('traceback')
load_initial_data = options.get('load_initial_data')
self.style = no_style()
# Import the 'management' module within each installed app, to register
# dispatcher events.
for app_name in settings.INSTALLED_APPS:
try:
import_module('.management', app_name)
except ImportError as exc:
# This is slightly hackish. We want to ignore ImportErrors
# if the "management" module itself is missing -- but we don't
# want to ignore the exception if the management module exists
# but raises an ImportError for some reason. The only way we
# can do this is to check the text of the exception. Note that
# we're a bit broad in how we check the text, because different
# Python implementations may not use the same text.
# CPython uses the text "No module named management"
# PyPy uses "No module named myproject.myapp.management"
msg = exc.args[0]
if not msg.startswith(
'No module named') or 'management' not in msg:
raise
db = options.get('database')
connection = connections[db]
cursor = connection.cursor()
# Get a list of already installed *models* so that references work right.
tables = connection.introspection.table_names()
seen_models = connection.introspection.installed_models(tables)
created_models = set()
pending_references = {}
# Build the manifest of apps and models that are to be synchronized
all_models = [(app.__name__.split('.')[-2], [
m for m in models.get_models(app, include_auto_created=True)
if router.allow_syncdb(db, m)
]) for app in models.get_apps()]
def model_installed(model):
opts = model._meta
converter = connection.introspection.table_name_converter
return not ((converter(opts.db_table) in tables) or
(opts.auto_created and converter(
opts.auto_created._meta.db_table) in tables))
manifest = SortedDict(
(app_name, list(filter(model_installed, model_list)))
for app_name, model_list in all_models)
# Create the tables for each model
if verbosity >= 1:
self.stdout.write("Creating tables ...\n")
for app_name, model_list in manifest.items():
for model in model_list:
# Create the model's database table, if it doesn't already exist.
if verbosity >= 3:
self.stdout.write("Processing %s.%s model\n" %
(app_name, model._meta.object_name))
sql, references = connection.creation.sql_create_model(
model, self.style, seen_models)
seen_models.add(model)
created_models.add(model)
for refto, refs in references.items():
pending_references.setdefault(refto, []).extend(refs)
if refto in seen_models:
sql.extend(
connection.creation.sql_for_pending_references(
refto, self.style, pending_references))
sql.extend(
connection.creation.sql_for_pending_references(
model, self.style, pending_references))
if verbosity >= 1 and sql:
self.stdout.write("Creating table %s\n" %
model._meta.db_table)
for statement in sql:
cursor.execute(statement)
tables.append(
connection.introspection.table_name_converter(
model._meta.db_table))
transaction.commit_unless_managed(using=db)
# Send the post_syncdb signal, so individual apps can do whatever they need
# to do at this point.
emit_post_sync_signal(created_models, verbosity, interactive, db)
# The connection may have been closed by a syncdb handler.
cursor = connection.cursor()
if verbosity >= 1:
self.stdout.write("Installing indexes ...\n")
# Install SQL indices for all newly created models
for app_name, model_list in manifest.items():
for model in model_list:
if model in created_models:
index_sql = connection.creation.sql_indexes_for_model(
model, self.style)
if index_sql:
if verbosity >= 2:
self.stdout.write(
"Installing index for %s.%s model\n" %
(app_name, model._meta.object_name))
try:
for sql in index_sql:
cursor.execute(sql)
except Exception as e:
self.stderr.write("Failed to install index for %s.%s model: %s\n" % \
(app_name, model._meta.object_name, e))
transaction.rollback_unless_managed(using=db)
else:
transaction.commit_unless_managed(using=db)
# Install custom SQL for the app (but only if this
# is a model we've just created)
if verbosity >= 1:
self.stdout.write("Installing custom SQL ...\n")
for app_name, model_list in manifest.items():
for model in model_list:
if model in created_models:
custom_sql = custom_sql_for_model(model, self.style,
connection)
if custom_sql:
if verbosity >= 2:
self.stdout.write(
"Installing custom SQL for %s.%s model\n" %
(app_name, model._meta.object_name))
try:
for sql in custom_sql:
cursor.execute(sql)
except Exception as e:
self.stderr.write("Failed to install custom SQL for %s.%s model: %s\n" % \
(app_name, model._meta.object_name, e))
if show_traceback:
traceback.print_exc()
transaction.rollback_unless_managed(using=db)
else:
transaction.commit_unless_managed(using=db)
else:
if verbosity >= 3:
self.stdout.write(
"No custom SQL for %s.%s model\n" %
(app_name, model._meta.object_name))
# Load initial_data fixtures (unless that has been disabled)
if load_initial_data:
call_command('loaddata',
'initial_data',
verbosity=verbosity,
database=db,
skip_validation=True)
class Command(NoArgsCommand):
option_list = NoArgsCommand.option_list + (make_option(
'--noinput',
action='store_false',
dest='interactive',
default=True,
help='Tells Django to NOT prompt the user for input of any kind.'), )
help = "Create the database tables for all apps in INSTALLED_APPS whose tables haven't already been created."
def handle_noargs(self, **options):
from django.db import connection, transaction, models
from django.conf import settings
from django.core.management.sql import custom_sql_for_model, emit_post_sync_signal
verbosity = int(options.get('verbosity', 1))
interactive = options.get('interactive')
show_traceback = options.get('traceback', False)
self.style = no_style()
# Import the 'management' module within each installed app, to register
# dispatcher events.
for app_name in settings.INSTALLED_APPS:
try:
__import__(app_name + '.management', {}, {}, [''])
except ImportError, exc:
# This is slightly hackish. We want to ignore ImportErrors
# if the "management" module itself is missing -- but we don't
# want to ignore the exception if the management module exists
# but raises an ImportError for some reason. The only way we
# can do this is to check the text of the exception. Note that
# we're a bit broad in how we check the text, because different
# Python implementations may not use the same text.
# CPython uses the text "No module named management"
# PyPy uses "No module named myproject.myapp.management"
msg = exc.args[0]
if not msg.startswith(
'No module named') or 'management' not in msg:
raise
cursor = connection.cursor()
# Get a list of already installed *models* so that references work right.
tables = connection.introspection.table_names()
seen_models = connection.introspection.installed_models(tables)
created_models = set()
pending_references = {}
# Create the tables for each model
for app in models.get_apps():
app_name = app.__name__.split('.')[-2]
model_list = models.get_models(app)
for model in model_list:
# Create the model's database table, if it doesn't already exist.
if verbosity >= 2:
print "Processing %s.%s model" % (app_name,
model._meta.object_name)
if connection.introspection.table_name_converter(
model._meta.db_table) in tables:
continue
sql, references = connection.creation.sql_create_model(
model, self.style, seen_models)
seen_models.add(model)
created_models.add(model)
for refto, refs in references.items():
pending_references.setdefault(refto, []).extend(refs)
if refto in seen_models:
sql.extend(
connection.creation.sql_for_pending_references(
refto, self.style, pending_references))
sql.extend(
connection.creation.sql_for_pending_references(
model, self.style, pending_references))
if verbosity >= 1:
print "Creating table %s" % model._meta.db_table
for statement in sql:
cursor.execute(statement)
tables.append(
connection.introspection.table_name_converter(
model._meta.db_table))
# Create the m2m tables. This must be done after all tables have been created
# to ensure that all referred tables will exist.
for app in models.get_apps():
app_name = app.__name__.split('.')[-2]
model_list = models.get_models(app)
for model in model_list:
if model in created_models:
sql = connection.creation.sql_for_many_to_many(
model, self.style)
if sql:
if verbosity >= 2:
print "Creating many-to-many tables for %s.%s model" % (
app_name, model._meta.object_name)
for statement in sql:
cursor.execute(statement)
transaction.commit_unless_managed()
# Send the post_syncdb signal, so individual apps can do whatever they need
# to do at this point.
emit_post_sync_signal(created_models, verbosity, interactive)
# The connection may have been closed by a syncdb handler.
cursor = connection.cursor()
# Install custom SQL for the app (but only if this
# is a model we've just created)
for app in models.get_apps():
app_name = app.__name__.split('.')[-2]
for model in models.get_models(app):
if model in created_models:
custom_sql = custom_sql_for_model(model, self.style)
if custom_sql:
if verbosity >= 1:
print "Installing custom SQL for %s.%s model" % (
app_name, model._meta.object_name)
try:
for sql in custom_sql:
cursor.execute(sql)
except Exception, e:
sys.stderr.write("Failed to install custom SQL for %s.%s model: %s\n" % \
(app_name, model._meta.object_name, e))
if show_traceback:
import traceback
traceback.print_exc()
transaction.rollback_unless_managed()
else:
transaction.commit_unless_managed()
else:
if verbosity >= 2:
print "No custom SQL for %s.%s model" % (
app_name, model._meta.object_name)
def delete(request,*kw,**kwargs):
if request.method == 'GET':
form = forms.DeleteForm()
return django.shortcuts.render_to_response('bots/delete.html', {'form': form},context_instance=django.template.RequestContext(request))
else:
if 'submit' in request.POST:
form = forms.DeleteForm(request.POST)
if form.is_valid():
botsglobal.logger.info(_(u'Start deleting in configuration.'))
if form.cleaned_data['deltransactions']:
#while testing with very big loads, deleting transaction when wrong. Using raw SQL solved this.
from django.db import connection, transaction
cursor = connection.cursor()
cursor.execute("DELETE FROM ta")
cursor.execute("DELETE FROM filereport")
cursor.execute("DELETE FROM report")
transaction.commit_unless_managed()
request.user.message_set.create(message=_(u'Transactions are deleted.'))
botsglobal.logger.info(_(u' Transactions are deleted.'))
#clean data files
deletefrompath = botsglobal.ini.get('directories','data','botssys/data')
shutil.rmtree(deletefrompath,ignore_errors=True)
botslib.dirshouldbethere(deletefrompath)
request.user.message_set.create(message=_(u'Data files are deleted.'))
botsglobal.logger.info(_(u' Data files are deleted.'))
if form.cleaned_data['delconfiguration']:
models.confirmrule.objects.all().delete()
models.channel.objects.all().delete()
models.chanpar.objects.all().delete()
models.partner.objects.all().delete()
models.translate.objects.all().delete()
models.routes.objects.all().delete()
request.user.message_set.create(message=_(u'Database configuration is deleted.'))
botsglobal.logger.info(_(u' Database configuration is deleted.'))
if form.cleaned_data['delcodelists']:
models.ccode.objects.all().delete()
models.ccodetrigger.objects.all().delete()
request.user.message_set.create(message=_(u'User code lists are deleted.'))
botsglobal.logger.info(_(u' User code lists are deleted.'))
if form.cleaned_data['delinfile']:
deletefrompath = botslib.join(botsglobal.ini.get('directories','botssys','botssys'),'infile')
shutil.rmtree('bots/botssys/infile',ignore_errors=True)
request.user.message_set.create(message=_(u'Files in botssys/infile are deleted.'))
botsglobal.logger.info(_(u' Files in botssys/infile are deleted.'))
if form.cleaned_data['deloutfile']:
deletefrompath = botslib.join(botsglobal.ini.get('directories','botssys','botssys'),'outfile')
shutil.rmtree('bots/botssys/outfile',ignore_errors=True)
request.user.message_set.create(message=_(u'Files in botssys/outfile are deleted.'))
botsglobal.logger.info(_(u' Files in botssys/outfile are deleted.'))
if form.cleaned_data['deluserscripts']:
deletefrompath = botsglobal.ini.get('directories','usersysabs')
for root, dirs, files in os.walk(deletefrompath):
head, tail = os.path.split(root)
if tail == 'charsets':
del dirs[:]
continue
for bestand in files:
if bestand != '__init__.py':
os.remove(os.path.join(root,bestand))
request.user.message_set.create(message=_(u'User scripts are deleted (in usersys).'))
botsglobal.logger.info(_(u' User scripts are deleted (in usersys).'))
elif form.cleaned_data['delbackup']:
deletefrompath = botsglobal.ini.get('directories','usersysabs')
for root, dirs, files in os.walk(deletefrompath):
head, tail = os.path.split(root)
if tail == 'charsets':
del dirs[:]
continue
for bestand in files:
name,ext = os.path.splitext(bestand)
if ext and len(ext) == 15 and ext[1:].isdigit() :
os.remove(os.path.join(root,bestand))
request.user.message_set.create(message=_(u'Backupped user scripts are deleted/purged (in usersys).'))
botsglobal.logger.info(_(u' Backupped user scripts are deleted/purged (in usersys).'))
botsglobal.logger.info(_(u'Finished deleting in configuration.'))
return django.shortcuts.redirect('/home')
def load_app(app_path):
testapp = django_load_app(app_path)
app_name = testapp.__name__.split('.')[-2]
connection = connections[DEFAULT_DB_ALIAS]
cursor = connection.cursor()
test_models = [
m for m in models.get_models(testapp, include_auto_created=True)
if router.allow_syncdb(DEFAULT_DB_ALIAS, m)
]
loaded_models[app_path] = test_models
# We assume the models haven't been installed, otherwise there's more to do here
# Get a list of already installed *models* so that references work right.
tables = connection.introspection.table_names()
seen_models = connection.introspection.installed_models(tables)
pending_references = {}
verbosity = 0
# Create the tables for each model
for model in test_models:
# Create the model's database table, if it doesn't already exist.
if verbosity >= 2:
print "Processing %s.%s model" % (app_name,
model._meta.object_name)
sql, references = connection.creation.sql_create_model(
model, no_style(), seen_models)
seen_models.add(model)
for refto, refs in references.items():
pending_references.setdefault(refto, []).extend(refs)
if refto in seen_models:
sql.extend(
connection.creation.sql_for_pending_references(
refto, no_style(), pending_references))
sql.extend(
connection.creation.sql_for_pending_references(
model, no_style(), pending_references))
if verbosity >= 1 and sql:
print "Creating table %s" % model._meta.db_table
for statement in sql:
cursor.execute(statement)
tables.append(
connection.introspection.table_name_converter(
model._meta.db_table))
transaction.commit_unless_managed(using=DEFAULT_DB_ALIAS)
for model in test_models:
index_sql = connection.creation.sql_indexes_for_model(
model, no_style())
if index_sql:
if verbosity >= 1:
print "Installing index for %s.%s model" % (
app_name, model._meta.object_name)
try:
for sql in index_sql:
cursor.execute(sql)
except Exception, e:
sys.stderr.write("Failed to install index for %s.%s model: %s\n" % \
(app_name, model._meta.object_name, e))
transaction.rollback_unless_managed(using=DEFAULT_DB_ALIAS)
else:
transaction.commit_unless_managed(using=DEFAULT_DB_ALIAS)
def handle(self, *fixture_labels, **options):
""" Main method of a Django command """
from django.db.models import get_apps
from django.core import serializers
from django.db import connection, transaction
from django.conf import settings
self.style = no_style()
verbosity = int(options.get('verbosity', 1))
show_traceback = options.get('traceback', False)
# Keep a count of the installed objects and fixtures
fixture_count = 0
object_count = 0
objects_per_fixture = []
models = set()
humanize = lambda dirname: dirname and "'%s'" % dirname or 'absolute path'
# Get a cursor (even though we don't need one yet). This has
# the side effect of initializing the test database (if
# it isn't already initialized).
cursor = connection.cursor()
# Start transaction management. All fixtures are installed in a
# single transaction to ensure that all references are resolved.
transaction.commit_unless_managed()
transaction.enter_transaction_management()
transaction.managed(True)
app_fixtures = [os.path.join(os.path.dirname(app.__file__), 'fixtures') \
for app in get_apps()]
for fixture_label in fixture_labels:
parts = fixture_label.split('.')
if len(parts) == 1:
fixture_name = fixture_label
formats = serializers.get_public_serializer_formats()
else:
fixture_name, format = '.'.join(parts[:-1]), parts[-1]
if format in serializers.get_public_serializer_formats():
formats = [format]
else:
formats = []
if formats:
if verbosity > 1:
print "Loading '%s' fixtures..." % fixture_name
else:
sys.stderr.write(
self.style.ERROR("Problem installing fixture '%s': %s is not a known " + \
"serialization format." % (fixture_name, format))
)
transaction.rollback()
transaction.leave_transaction_management()
return
if os.path.isabs(fixture_name):
fixture_dirs = [fixture_name]
else:
fixture_dirs = app_fixtures + list(
settings.FIXTURE_DIRS) + ['']
for fixture_dir in fixture_dirs:
if verbosity > 1:
print "Checking %s for fixtures..." % humanize(fixture_dir)
label_found = False
for format in formats:
serializer = serializers.get_serializer(format)
if verbosity > 1:
print "Trying %s for %s fixture '%s'..." % \
(humanize(fixture_dir), format, fixture_name)
try:
full_path = os.path.join(
fixture_dir, '.'.join([fixture_name, format]))
fixture = open(full_path, 'r')
if label_found:
fixture.close()
print self.style.ERROR(
"Multiple fixtures named '%s' in %s. Aborting."
% (fixture_name, humanize(fixture_dir)))
transaction.rollback()
transaction.leave_transaction_management()
return
else:
fixture_count += 1
objects_per_fixture.append(0)
if verbosity > 0:
print "Installing %s fixture '%s' from %s." % \
(format, fixture_name, humanize(fixture_dir))
try:
objects_to_keep = {}
objects = serializers.deserialize(
format, fixture)
for obj in objects:
object_count += 1
objects_per_fixture[-1] += 1
class_ = obj.object.__class__
if not class_ in objects_to_keep:
objects_to_keep[class_] = set()
objects_to_keep[class_].add(obj.object)
models.add(class_)
obj.save()
self.remove_objects_not_in(
objects_to_keep, verbosity)
label_found = True
except (SystemExit, KeyboardInterrupt):
raise
except Exception:
import traceback
fixture.close()
transaction.rollback()
transaction.leave_transaction_management()
if show_traceback:
traceback.print_exc()
else:
sys.stderr.write(
self.style.ERROR(
"Problem installing fixture '%s': %s\n"
% (full_path,
traceback.format_exc())))
return
fixture.close()
except:
if verbosity > 1:
print "No %s fixture '%s' in %s." % \
(format, fixture_name, humanize(fixture_dir))
# If any of the fixtures we loaded contain 0 objects, assume that an
# error was encountered during fixture loading.
if 0 in objects_per_fixture:
sys.stderr.write(
self.style.ERROR(
"No fixture data found for '%s'. (File format may be invalid.)"
% (fixture_name)))
transaction.rollback()
transaction.leave_transaction_management()
return
# If we found even one object in a fixture, we need to reset the
# database sequences.
if object_count > 0:
sequence_sql = connection.ops.sequence_reset_sql(
self.style, models)
if sequence_sql:
if verbosity > 1:
print "Resetting sequences"
for line in sequence_sql:
cursor.execute(line)
transaction.commit()
transaction.leave_transaction_management()
if object_count == 0:
if verbosity > 1:
print "No fixtures found."
else:
if verbosity > 0:
print "Installed %d object(s) from %d fixture(s)" % (
object_count, fixture_count)
# Close the DB connection. This is required as a workaround for an
# edge case in MySQL: if the same connection is used to
# create tables, load data, and query, the query can return
# incorrect results. See Django #7572, MySQL #37735.
connection.close()
