def on_config_entity_post_save_layer(sender, **kwargs): """ Sync tilestache to a ConfigEntity class after the latter is saved :param **kwargs: optional "db_entity_keys" to limit the layers created to those DbEntities """ from footprint.client.configuration.fixture import LayerConfigurationFixture from footprint.client.configuration.utils import resolve_fixture # Disable post save publishing on individual layers. The ConfigEntity is controlling publishing config_entity = InstanceBundle.extract_single_instance(**kwargs) logger.info("Handler: on_config_entity_post_save_layer for %s" % config_entity.full_name) # Create LayerLibrary instances based on each LayerLibrary configuration if the configuration's scope # matches that of config_entity client_layer_fixture = resolve_fixture( "presentation", "layer", LayerConfigurationFixture, config_entity.schema(), config_entity=config_entity) layer_library_configurations = FixtureList(client_layer_fixture.layer_libraries()).matching_scope( class_scope=config_entity.__class__ ) logger.info("Processing LayerLibrary Configurations %s" % ', '.join(map( lambda layer_library_configuration: layer_library_configuration.key, layer_library_configurations))) for layer_library_configuration in layer_library_configurations: _update_or_create_layer_library_and_layers(config_entity, layer_library_configuration, **kwargs) reset_queries()
def import_consumption(): data_reader = csv.reader(open(SOURCE_FILE)) insert_count = 0 cutoff_year = 1990 for i, row in enumerate(data_reader): if i==0: years = row[2:] #if we already have data for the years in the file, delete it for h, header in enumerate(years): if header >= cutoff_year: print 'deleting %s records' % header data = EnergyConsumptionStateRaw.objects.filter(year=header) data.delete() else: for j, col in enumerate(row[2:]): year = years[j] if year >= cutoff_year: record = EnergyConsumptionStateRaw( year=year, state=row[0], msn=row[1], value=clean_num(col)) record.save() db.reset_queries() insert_count = insert_count + 1 print '%s import complete. %s records inserted' % ( SOURCE_FILE, insert_count)
def show_queries(db_alias=None, sqlparse_character_limit=2048): old_debug_setting = settings.DEBUG try: settings.DEBUG = True # This call to reset_queries ensures that the query list is # empty before running the wrapped code, and stops the query # log from just getting bigger and bigger if this context # manager is used repeatedly. reset_queries() yield queries_after = get_queries(db_alias)[:] number_of_queries = len(queries_after) print("--===--") print("Number of queries: {n}".format(n=number_of_queries)) for i, q in enumerate(queries_after): query_time = q['time'] query_sql = q['sql'] query_length = len(query_sql) print(" Query {i} (taking {t}): ".format(i=i, t=query_time)) # Outputting the formatted query takes a very long time # for large queries (e.g. those that prefetch_related can # generate with "IN (... thousands of IDs ...)"), so only # pretty-print queries that are fairly short. if SQLPARSE_AVAILABLE and query_length <= sqlparse_character_limit: formatted = sqlparse.format( query_sql, reindent=True, keyword_case='upper') print(indent(formatted, 4)) else: print(indent(query_sql, 4)) print("End of query output.") finally: settings.DEBUG = old_debug_setting
def test_subselect_support(self): """Test that subselects are handled properly.""" from django import db db.reset_queries() from testapp.models import Book, Person, PersonType author_types = PersonType.objects.filter(title='Author') author_people = Person.objects.filter(person_types__in=author_types) written_books = Book.objects.filter(authors__in=author_people) q = base.message_queue() self.failUnless(len(db.connection.queries) == 0) count = written_books.count() self.failUnless(q.get() == False) # execute the query again, this time it's cached self.failUnless(written_books.count() == count) self.failUnless(q.get() == True) # change the person type of 'Author' to something else pt = PersonType.objects.get(title='Author') pt.title = 'NonAuthor' pt.save() self.failUnless(PersonType.objects.filter(title='Author').count() == 0) q.clear() db.reset_queries() # now execute the same query; the result should be diff and it should be # a cache miss new_count = written_books.count() self.failUnless(new_count != count) self.failUnless(q.get() == False) PersonType.objects.filter(title='NonAuthor').order_by('-title')[:5]
def handle_noargs(self, **options): def clean_int(value): if value=='': value=None return value data_reader = csv.reader(open(SOURCE_FILE)) for i, row in enumerate(data_reader): if i == 0: year_row = row; else: state = row[0] agency_name = row[1] agency_id = row[2] for j,col in enumerate(row): if j > 2: record = DrugFreeSchoolSpending() record.year = year_row[j] record.state = state record.agency_name = agency_name record.agency_id = agency_id record.amount = clean_int(col) record.save() db.reset_queries()
def _get_user(self, user, password=None): """Gets specified user from Django's userdb If setting django_required_group is defined, user MUST be in that group If password is specified, also checks it. Returns User object if user is found (optionally: AND belongs to specified group) (optionally: AND if password is correct) Returns None if user is not found OR error occurs Returns False if user is found, but password is incorrect OR user doesn't belong to required group """ db.reset_queries() try: try: duser = User.objects.get(Q(is_active=True) & \ (Q(**{username_field: user}) | Q(email=user))) group = str(self.require_group) if group != "": if duser.groups.filter(name=group).count() == 0: return False if password and duser.check_password(password): return duser elif password is None: return duser else: return False except User.DoesNotExist: return None finally: db.connection.close() return None
def run_main_task(entry_id, task_fcn, action_name): """ Applies the `task_fcn` to the arguments defined in `entry_id` InstructorTask. Arguments passed to `task_fcn` are: `entry_id` : the primary key for the InstructorTask entry representing the task. `course_id` : the id for the course. `task_input` : dict containing task-specific arguments, JSON-decoded from InstructorTask's task_input. `action_name` : past-tense verb to use for constructing status messages. If no exceptions are raised, the `task_fcn` should return a dict containing the task's result with the following keys: 'attempted': number of attempts made 'succeeded': number of attempts that "succeeded" 'skipped': number of attempts that "skipped" 'failed': number of attempts that "failed" 'total': number of possible subtasks to attempt 'action_name': user-visible verb to use in status messages. Should be past-tense. Pass-through of input `action_name`. 'duration_ms': how long the task has (or had) been running. """ # Get the InstructorTask to be updated. If this fails then let the exception return to Celery. # There's no point in catching it here. with outer_atomic(): entry = InstructorTask.objects.get(pk=entry_id) entry.task_state = PROGRESS entry.save_now() # Get inputs to use in this task from the entry task_id = entry.task_id course_id = entry.course_id task_input = json.loads(entry.task_input) # Construct log message fmt = u'Task: {task_id}, InstructorTask ID: {entry_id}, Course: {course_id}, Input: {task_input}' task_info_string = fmt.format(task_id=task_id, entry_id=entry_id, course_id=course_id, task_input=task_input) TASK_LOG.info(u'%s, Starting update (nothing %s yet)', task_info_string, action_name) # Check that the task_id submitted in the InstructorTask matches the current task # that is running. request_task_id = _get_current_task().request.id if task_id != request_task_id: fmt = u'{task_info}, Requested task did not match actual task "{actual_id}"' message = fmt.format(task_info=task_info_string, actual_id=request_task_id) TASK_LOG.error(message) raise ValueError(message) # Now do the work task_progress = task_fcn(entry_id, course_id, task_input, action_name) # Release any queries that the connection has been hanging onto reset_queries() # Log and exit, returning task_progress info as task result TASK_LOG.info(u'%s, Task type: %s, Finishing task: %s', task_info_string, action_name, task_progress) return task_progress
def ideas_api_to_db(**kwargs): # get non-ER ideas ideas=get_ideas_from_api(**kwargs) for idea in ideas: write_idea_to_db(idea, accessed_at = timezone.now()) db.reset_queries() stats.update()
def _check_name_deferred(self, batch): entries = Entry.objects.batch_select(batch).order_by('id') entries = list(entries) self.failUnlessEqual([self.entry1, self.entry2, self.entry3, self.entry4], entries) self.failUnlessEqual(2, len(db.connection.queries)) db.reset_queries() entry1, entry2, entry3, entry4 = entries self.failUnlessEqual(3, len(entry1.tags_all)) self.failUnlessEqual(1, len(entry2.tags_all)) self.failUnlessEqual(2, len(entry3.tags_all)) self.failUnlessEqual(0, len(entry4.tags_all)) self.failUnlessEqual(0, len(db.connection.queries)) # as name has been defered it should trigger a query when we # try to access it self.failUnlessEqual( self.tag2.name, entry1.tags_all[0].name ) self.failUnlessEqual(1, len(db.connection.queries)) self.failUnlessEqual( self.tag1.name, entry1.tags_all[1].name ) self.failUnlessEqual(2, len(db.connection.queries)) self.failUnlessEqual( self.tag3.name, entry1.tags_all[2].name ) self.failUnlessEqual(3, len(db.connection.queries))
def read_year(file): def nullify(value): if len(value): return value else: return None with open(file) as f: reader = csv.reader(f) oldrecs = UsaspendingAssistanceRaw.objects.filter(fiscal_year=year, asst_cat_type = file.split('/')[-1][0]) print 'cleaning up old records....' reccount = oldrecs.count() oldrecs.delete() print 'deleted %s records.' % reccount insert_count = 0 for i, row in enumerate(reader): if i == 0: header_row = row else: record = UsaspendingAssistanceRaw() for j,col in enumerate(row): setattr(record, header_row[j], nullify(col)) record.save() db.reset_queries() insert_count = insert_count + 1 print '%s done: %s records inserted.' % (file, insert_count)
def callback(result): attempt = 0 retry_limit = self.config.get_exponential_limit() while attempt < retry_limit: self.stats['timestamp'] = self._utcnow() try: (verified, exist) = result if verified: self.send_verified_notification( exist, conn, exchange, routing_keys=routing_keys) break except exceptions.ObjectDoesNotExist: if attempt < retry_limit-1: logger.warn("ObjectDoesNotExist in callback, " "attempting to reconnect and try " "again.") close_connection() reset_queries() else: logger.error("ObjectDoesNotExist in callback " "again, giving up.") # Avoiding unnecessary sleep() break except librabbitmq.ConnectionError as e: logger.error("ConnectionEror found while trying to connect to RabbitMQ. \ Attempting the {}th time.".format(attempt)) except Exception, e: msg = "ERROR in Callback %s: %s" % (exchange_name, e) logger.exception(msg) break attempt += 1 # Exponentially timed backoff time.sleep((2 ** attempt) / 1000.0 + (random.randint(0, 1000) / 1000.0))
def handle_noargs(self, **options): def clean_int(value): if value.strip()=='': value=None else: value=int(value) return value data_reader = csv.reader(open(SOURCE_FILE)) for i, row in enumerate(data_reader): if i == 0: year_row = row; else: place = row[0] state = row[1] type = row[2] for j,col in enumerate(row): if j > 2: record = WICParticipants() record.year = int(year_row[j]) record.place = place record.state = state record.value = clean_int(col) record.save() db.reset_queries()
def blog_archive(self): db.reset_queries() ctx = { 'request': self.get_full_path(), 'items': Article.objects.filter(publish=True).filter(published__lt=datetime.now()) } return render_to_response('archive.html', ctx)
def home(self): db.reset_queries() ctx = { 'request': self.get_full_path(), 'items': Article.objects.filter(publish=True)[:4], } return render_to_response('base.html', ctx)
def write(self, impressions): insert_sql = "INSERT INTO bannerimpressions (timestamp, banner, campaign, project_id, language_id, country_id, count) VALUES (%s) ON DUPLICATE KEY update count=count+%d" cursor = connections['default'].cursor() if not len(impressions): return try: for k,c in impressions.iteritems(): try: cursor.execute(insert_sql % ( "%s, %d" % (k, c), c )) except (MySQLdb.Warning, _mysql_exceptions.Warning) as e: pass # We don't care about the message transaction.commit('default') except Exception as e: import sys transaction.rollback("default") self.logger.exception("UNHANDLED EXCEPTION: %s" % str(e)) self.logger.exception(sys.exc_info()[0]) if self.debug: if len(impressions) == 1: self.logger.info(impressions) for r in self.debug_info: self.logger.info("\t%s" % r) finally: reset_queries() del impressions del cursor
def show_page(self, url): db.reset_queries() ctx = { 'request': self.get_full_path(), 'item': get_object_or_404(Page, slug=url, state=True) } return render_to_response('page.html', ctx)
def groups_for_user(environ, username): """ Authorizes a user based on groups """ # We cannot load these modules before calling get_wsgi_application() # So they must be defined here. from ngw.core import perms from ngw.core.models import ContactGroup UserModel = auth.get_user_model() db.reset_queries() try: try: user = UserModel._default_manager.get_by_natural_key(username) except UserModel.DoesNotExist: return [] if not user.is_active: return [] groups = ContactGroup.objects.with_user_perms( user.id, wanted_flags=perms.VIEW_FILES, add_column=False) return [force_bytes(group.id) for group in groups] finally: db.close_old_connections()
def index_titles(since=None): """index all the titles and holdings that are modeled in the database if you pass in a datetime object as the since parameter only title records that have been created since that time will be indexed. """ cursor = connection.cursor() solr = SolrConnection(settings.SOLR) if since: cursor.execute("SELECT lccn FROM core_title WHERE created >= '%s'" % since) else: solr.delete_query('type:title') cursor.execute("SELECT lccn FROM core_title") count = 0 while True: row = cursor.fetchone() if row is None: break title = models.Title.objects.get(lccn=row[0]) index_title(title, solr) count += 1 if count % 100 == 0: LOGGER.info("indexed %s titles", count) reset_queries() solr.commit() solr.commit()
def run_policy_once(): from core.models import Instance,Slice,Controller,Network,User,SlicePrivilege,Site,SitePrivilege,Image,ControllerSlice,ControllerUser,ControllerSite models = [Controller, Site, SitePrivilege, Image, ControllerSlice, ControllerSite, ControllerUser, User, Slice, Network, Instance, SlicePrivilege] objects = [] deleted_objects = [] for m in models: res = m.objects.filter((Q(policed__lt=F('updated')) | Q(policed=None)) & Q(no_policy=False)) objects.extend(res) res = m.deleted_objects.filter(Q(policed__lt=F('updated')) | Q(policed=None)) deleted_objects.extend(res) for o in objects: execute_model_policy(o, o.deleted) for o in deleted_objects: execute_model_policy(o, True) # Reap non-sync'd models here reaped = [Slice] for m in reaped: dobjs = m.deleted_objects.all() for d in dobjs: deps = walk_inv_deps(noop, d) if (not deps): print 'Purging object %r'%d d.delete(purge=True) try: reset_queries() except: # this shouldn't happen, but in case it does, catch it... logger.log_exc("exception in reset_queries")
def __exit__(self, exc_type, exc_val, exc_tb): self.time_total = default_timer() - self.start_time if self.db_profile: settings.DEBUG = self.old_debug self.queries = connection.queries reset_queries()
def process_url(self, records_url, options): total_records = self.calculate_records_nb(records_url) writer = None errors=[] context = ET.iterparse(records_url, events=("end",)) i = 0 for _,elem in context: if elem.tag == "{%s}Record" % IIEP: i += 1 writer = show_progress(i, total_records, "Processing record nb %d " % i, 40, writer=writer, newline=self.newline) try: record_graph = get_empty_graph() record_graph.parse(data=ET.tostring(elem, encoding='utf-8'), format='xml') self.record_parser.build_record(record_graph, delete=(not self.preserve)) except Exception as e: transaction.rollback() msg = "Error processing resource %d in %s : %s" % (i, records_url, repr(e)) logger.exception(msg) errors.append((i, records_url, msg)) else: transaction.commit() if i%self.batch_size == 0: reset_queries() return errors
def handle_noargs(self, **options): state_name = "" total_inserts = 0 total_updates = 0 raw = SnapParticipationPeopleStateRaw.objects.all().order_by("state") total_raw = raw.count() for r in raw: if r.state != state_name: clean_state = clean_state_name(r.state) try: state_ref_current = State.objects.get(state_name__iexact=clean_state) except: print "Skipping record. Unable to find state: " + clean_state continue state_name = r.state try: record = SnapParticipationPeopleState.objects.get(year=r.year, state=state_ref_current) total_updates = total_updates + 1 except: record = SnapParticipationPeopleState(year=r.year, state=state_ref_current) total_inserts = total_inserts + 1 record.value = r.value record.save() db.reset_queries() print "SNAP Participation (state): total records from raw data = %s" % total_raw print "SNAP Participation (state): total inserts = %s" % total_inserts print "SNAP Participation (state): total updates = %s" % total_updates
def callback(result): attempt = 0 while attempt < 2: try: (verified, exist) = result if verified: self.send_verified_notification( exist, conn, exchange, routing_keys=routing_keys) break except exceptions.ObjectDoesNotExist: if attempt < 1: logger.warn("ObjectDoesNotExist in callback, " "attempting to reconnect and try " "again.") close_connection() reset_queries() else: logger.error("ObjectDoesNotExist in callback " "again, giving up.") except Exception, e: msg = "ERROR in Callback %s: %s" % (exchange_name, e) logger.exception(msg) break attempt += 1
def doit(self, options): if len(self.thelist): lstndis = [] Fake = options["fake"] if options["client"]: for cl in options["client"].split(","): try: lstndiclient = Client.objects.get(code=cl).sipaccount_set.all().values_list("cli", flat=True) logger.debug("NDIS CLIENT (%s) %s" % (cl, lstndiclient)) lstndis += lstndiclient except: logger.error("NO NDI FOR CLIENT %s" % cl) logger.debug("LIST NDIS %s" % lstndis) for filename in self.thelist: logger.info("FILENAME %s" % filename) if not Fake: try: self.cdrs += ie_Cdr().import_cdr( workdate=self.workdate, fromtelco=self.telco, injector=self.telcoinjector, filename=filename, force=options["force"], clientndis=lstndis, check=options["check"], ) except ie_Exception, e: logger.error("%s %s" % (e, filename)) if self.djangodebug: logger.debug("Garbage Django Debug") db.reset_queries()
def batchIndex(self): # index everything in db that comes after the last thing we indexed on # the previous run print '--> batch indexing %s' % self.valueCode while 1: recs = self.queryManager.getData(minTime=self.status['maxTime']) n = recs.count() if n == 0: break print '--> %d %s samples remaining' % (n, self.valueCode) for rec in recs[:BATCH_READ_NUM_SAMPLES]: self.indexRecord(rec) # avoid django debug log memory leak db.reset_queries() # batch process new records that arrived while we were # processing the database table. print ('--> indexing %d %s samples that came in during batch indexing' % (len(self.queue), self.valueCode)) self.flushQueue() # switch modes to process each new record as it comes in. print '--> switching to live data mode' self.queueMode = False
def loop(self): """ Main loop. """ logging.debug('Main loop started!') try: while self._running or self.events: t0 = time() events = self.events self.events = [] if events: t1 = time() logging.info('processing %d events...' % len(events)) with transaction.commit_on_success(): if events: for event in events: self.handle_event(event) reset_queries() logging.info('processed %d events in %.1f seconds' % (len(events), time() - t1)) if self._running and t0 < time(): sleep(self.interval - time() % self.interval) except: traceback.print_exc() self.stop()
def process_raw_data(deployment, args, json_args): """This is called directly by the worker to add the event to the db.""" db.reset_queries() routing_key, body = args record = None handler = HANDLERS.get(routing_key, None) if handler: values = handler(routing_key, body) if not values: return record values['deployment'] = deployment try: when = body['timestamp'] except KeyError: when = body['_context_timestamp'] # Old way of doing it values['when'] = utils.str_time_to_unix(when) values['routing_key'] = routing_key values['json'] = json_args record = STACKDB.create_rawdata(**values) STACKDB.save(record) aggregate_lifecycle(record) aggregate_usage(record, body) return record
def process_request(self, request): """ Setup the profiler for a profiling run and clear the SQL query log. If this is a resort of an existing profiling run, just return the resorted list. """ def unpickle(params): stats = unpickle_stats(b64decode(params.get('stats', ''))) queries = cPickle.loads(b64decode(params.get('queries', ''))) return stats, queries if request.method != 'GET' and \ not (request.META.get('HTTP_CONTENT_TYPE', request.META.get('CONTENT_TYPE', '')) in ['multipart/form-data', 'application/x-www-form-urlencoded']): return if (request.REQUEST.get('profile', False) and (settings.DEBUG == True or request.user.is_staff)): request.statsfile = tempfile.NamedTemporaryFile() params = request.REQUEST if (params.get('show_stats', False) and params.get('show_queries', '1') == '1'): # Instantly re-sort the existing stats data stats, queries = unpickle(params) return display_stats(request, stats, queries) elif (params.get('show_queries', False) and params.get('show_stats', '1') == '1'): stats, queries = unpickle(params) return display_queries(request, stats, queries) else: # We don't have previous data, so initialize the profiler request.profiler = hotshot.Profile(request.statsfile.name) reset_queries()
def import_data(self, metadata, output): entity_type, source = self._get_entity_type(), self._get_source() if not os.path.exists(self.codepoint_path): self._download_codepoint_open() try: archive = zipfile.ZipFile(self.codepoint_path) except zipfile.BadZipfile: self._download_codepoint_open() archive = zipfile.ZipFile(self.codepoint_path) if self.import_areas: filenames = ['Code-Point Open/Data/%s.csv' % code.lower() for code in self.import_areas] else: filenames = [path for path in archive.namelist() if re.match(r'Code\-Point Open\/Data\/[a-z]{1,2}.csv', path)] for filename in filenames: reset_queries() with transaction.commit_on_success(): if hasattr(archive, 'open'): f = archive.open(filename) else: f = tempfile.TemporaryFile() f.write(archive.read(filename)) f.seek(0) reader = csv.reader(f) self._load_from_csv(reader, entity_type, source) del f
def check_password(environ, username, password): """ Authenticates against Django's auth database mod_wsgi docs specify None, True, False as return value depending on whether the user exists and authenticates. """ UserModel = auth.get_user_model() # db connection state is managed similarly to the wsgi handler # as mod_wsgi may call these functions outside of a request/response cycle db.reset_queries() try: try: user = UserModel.objects.get_by_natural_key(username) except UserModel.DoesNotExist: return None try: if not user.is_active: return None except AttributeError as e: # a custom user may not support is_active return None return user.check_password(password) finally: db.close_connection()
def inner_func(*args, **kwargs): reset_queries() start_queries = len(connection.queries) start = time.perf_counter() result = func(*args, **kwargs) end = time.perf_counter() end_queries = len(connection.queries) print(f"Function : {func.__name__}") print(f"Number of Queries : {end_queries - start_queries}") print(f"Finished in : {(end - start):.2f}s") for query in connection.queries: print("SQl = {}".format(query['sql'])) return result
def xmlrpc_next_to_archive(self): '''Returns the meta information of all the file servers currently in the database''' #IP_ADDRESS = socket.gethostbyname(socket.gethostname()) cur_loc = get_current_location('') isConf, params = get_params(cur_loc) retdict = {} if isConf: backupFreeSpace, bkpPerFree, removeOnly = get_archive_report( params) serverFullSpace = get_server_full_space(cur_loc) for serverpath, space in serverFullSpace: retdict[serverpath] = build_exp_list( cur_loc, params['NUMBER_TO_BACKUP'], serverpath, removeOnly) db.reset_queries() return retdict
def do_update(backend, index, qs, start, end, total, verbosity=1): # Get a clone of the QuerySet so that the cache doesn't bloat up # in memory. Useful when reindexing large amounts of data. small_cache_qs = qs.all() current_qs = small_cache_qs[start:end] if verbosity >= 2: if hasattr(os, 'getppid') and os.getpid() == os.getppid(): print(" indexed %s - %d of %d." % (start + 1, end, total)) else: print(" indexed %s - %d of %d (by %s)." % (start + 1, end, total, os.getpid())) # FIXME: Get the right backend. backend.update(index, current_qs) # Clear out the DB connections queries because it bloats up RAM. reset_queries()
def _reindex_queryset(cls, queryset, **kwargs): """ Private classmethod to reindex a queryset. """ for key, obj in tqdm( enumerate(queryset), desc=cls.__name__, unit='doc', total=len(queryset), ): obj.es_index(**kwargs) if key % 100 == 0 and key != 0: """ Reset Queries to stop django Debug killing large reindexes. """ db.reset_queries()
def groups_for_user(environ, username): """ Authorizes a user based on groups """ db.reset_queries() try: try: user = UserModel._default_manager.get_by_natural_key(username) except UserModel.DoesNotExist: return [] if not user.is_active: return [] return [force_bytes(group.name) for group in user.groups.all()] finally: db.close_old_connections()
def address(): global db, pk_cache # 0 modelname = 'gw_addrshort' log(modelname) for i in (db.get_idlist(modelname)): v = db.get_fields(modelname, i[0]) pk_cache.set(modelname, i[0], AddrShort.objects.create( name = v['name'], fullname = v['fullname'] ).pk) transaction.commit() reset_queries() # 1 modelname = 'gw_addrtype' log(modelname) for i in (db.get_idlist(modelname)): v = db.get_fields(modelname, i[0]) pk_cache.set(modelname, i[0], AddrType.objects.create( abbr = v['abbr'], name = v['name'] ).pk) transaction.commit() reset_queries() # 2 modelname = 'gw_address' log(modelname) for n, i in enumerate(db.get_idplist(modelname)): v = db.get_fields(modelname, i[0]) pk_cache.set(modelname, i[0], Address.objects.create( name = v['name'], type_id = pk_cache.get('gw_addrshort', int(v['type_id'])) if v['type_id'] else None, typeplace = v['typeplace'], parent_id = pk_cache.get(modelname, long(v['parent_id'])) if v['parent_id'] else None, publish = v['publish'], endpoint = v['endpoint'], zip = v['zip'], fullname = v['fullname'] ).pk) if (n % 100000 == 99999): log("Commiting %d" % (n + 1)) transaction.commit() log("Commited") transaction.commit() reset_queries() # 3 modelname = 'gw_addrkladr' log(modelname) for i in (db.get_idplist(modelname)): v = db.get_fields(modelname, i[0]) pk_cache.set(modelname, i[0], AddrKladr.objects.create( address_id = pk_cache.get('gw_address', long(v['address_id'])), kladr_id = v['kladr_id'] ).pk) transaction.commit() reset_queries()
def tick(self): """ Called every second. Performs periodic maintainance and runs pending Map/Reduce tasks """ t = time.time() reset_queries() # Clear debug SQL log if self.batched_events: self.logger.info("Writing %d batched events", self.batched_events) self.event_batch.execute({"w": 0}) self.prepare_event_bulk() if t - self.last_mrtask_check >= self.mrt_schedule_interval: # Check Map/Reduce task status self.process_mrtasks() self.last_mrtask_check = t if t - self.last_status_refresh >= self.activator_status_interval: self.refresh_activator_status()
def test_queries(self): """ Test the documented API of connection.queries. """ reset_queries() with connection.cursor() as cursor: cursor.execute("SELECT 1" + connection.features.bare_select_suffix) self.assertEqual(1, len(connection.queries)) self.assertIsInstance(connection.queries, list) self.assertIsInstance(connection.queries[0], dict) six.assertCountEqual(self, connection.queries[0].keys(), ['sql', 'time']) reset_queries() self.assertEqual(0, len(connection.queries))
def test_many_to_one_mapping_cache_with_delete(self): """ Cache should be invalidated when calling 'delete' on related objects """ car2 = CarFactory.create(make=self.manufacturer) initial_count = len( Manufacturer.objects.get(id=self.manufacturer.id).cars.all()) car2.delete() reset_queries() # Only 1 cache (the one for car selection query) will be invalidated # as we only delete data on Car table new_count = len( Manufacturer.objects.get(id=self.manufacturer.id).cars.all()) # Because of m2m fix the no. of queries will be 2 instead of 1 self.assertEqual(len(connection.queries), 2) self.assertEqual(initial_count - 1, new_count)
def import_asnum_data(all_asnums): t_start = time.mktime(datetime.datetime.now().timetuple()) sys.stdout.write("* commiting asnums to database: ") sys.stdout.flush() for asn in all_asnums: asnum = ASNum( asnum=asn[0], name=asn[1], country=asn[2], rir=asn[3], regdate=asn[4], ) asnum.save() transaction.commit() reset_queries() t_end = time.mktime(datetime.datetime.now().timetuple()) print "%s seconds" % (int(t_end - t_start))
def run_policy(): from core.models import Sliver, Slice, Controller, Network, User, SlicePrivilege, Site, SitePrivilege, Image, ControllerSlice, ControllerUser, ControllerSite while (True): start = time.time() models = [ Sliver, Slice, Controller, Network, User, SlicePrivilege, Site, SitePrivilege, Image, ControllerSlice, ControllerSite, ControllerUser ] objects = [] deleted_objects = [] for m in models: res = m.objects.filter( Q(policed__lt=F('updated')) | Q(policed=None)) objects.extend(res) res = m.deleted_objects.filter( Q(policed__lt=F('updated')) | Q(policed=None)) deleted_objects.extend(res) for o in objects: execute_model_policy(o, o.deleted) for o in deleted_objects: execute_model_policy(o, True) # Reap non-sync'd models here reaped = [Slice] for m in reaped: dobjs = m.deleted_objects.all() for d in dobjs: deps = walk_inv_deps(noop, d) if (not deps): print 'Purging object %r' % d d.delete(purge=True) try: reset_queries() except: # this shouldn't happen, but in case it does, catch it... logger.log_exc("exception in reset_queries") if (time.time() - start < 1): time.sleep(1)
def handle_noargs(self, **options): data_reader = csv.reader(open(SOURCE_FILE)) insert_count = 0 update_count = 0 unchanged_count = 0 for i, row in enumerate(data_reader): if i == 0: year_row = row else: state = row[0] if len(state): for j, col in enumerate(row): if j > 0: try: #if year & state already exist, update the value (previous years' data is often revised) record = SnapMonthlyBenefitsPersonStateRaw.objects.get( state=state, year=int(year_row[j])) current_value = clean_num(col) if record.value != current_value: record.value = current_value record.save() update_count = update_count + 1 else: unchanged_count = unchanged_count + 1 except MultipleObjectsReturned: print 'error: multiple records exist for %s %s' % ( year_row[j], state) continue except: #this year & state isn't in the db yet; insert record = SnapMonthlyBenefitsPersonStateRaw() record.year = int(year_row[j]) record.state = state record.value = clean_num(col) record.save() insert_count = insert_count + 1 db.reset_queries() print 'snap monthly benefits import complete. %s inserted, %s updated, %s unchanged' % ( insert_count, update_count, unchanged_count)
def test_get_editable_channel_query_performance(self): """ Test that we are not running too many queries in order to return a single editable channel result. """ settings.DEBUG = True self.channel.editors.add(self.user) self.channel.save() reset_queries() request = self.create_get_request(reverse('get_user_edit_channels')) response = get_user_edit_channels(request) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) # This is a warning sign for performance problems, so if the number of queries goes above this # number, we need to evaluate the change and see if we can do something to optimize. self.assertQueriesLessThan(10)
def test_populate_db_ops(self): """Sanity check for the number of DB operations and records created.""" publication = CookbookPublication.objects.create( repository_version=self.version1) reset_queries() entries = list( populate(publication, batch_size=int(self.content_count / 2) + 1)) # 1 query for content count (batch_qs generator) -> 2 batches # 2 for content # 2 for content_artifact prefetch # 2 for bulk create of published artifacts self.assertLessEqual( len(connection.queries), 7, msg=f"More than 7 queries:\n {connection.queries}") self.assertEqual(len(entries), self.content_count) self.assertEqual(PublishedArtifact.objects.count(), self.content_count)
def youtube_daemon(self): x = 0 while True: self.course_list = generate_course_list() for course in self.course_list: LOGGER.info('%s%s: Callback' % (course.institution, course.edx_classid)) callfunction(course) x += 1 if x >= 100: LOGGER.info('Memory usage: %s (kb)' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) x = 0 reset_queries() self.course_list = [] time.sleep(10)
def run(self): os.umask(022) while True: ddb.reset_queries() jobs = queue.Job.objects.filter(state=1, account__host__state=1, created__lt=datetime.now() - timedelta(seconds=-200000000)) for job in jobs: try: check_die() if job.is_done(): jlogger.info('Collected %s' % job) job.collect() except: jlogger.warn('Job collection error!') break check_die(20)
def inner_func(*args, **kwargs): reset_queries() start_queries = len(connection.queries) start = time.perf_counter() result = func(*args, **kwargs) end = time.perf_counter() end_queries = len(connection.queries) print(f"Function : {func.__name__}") print(f"Number of Queries : {end_queries - start_queries}") print(f"Finished in : {(end - start):.2f}s") function_name = 'Function : {}'.format(func.__name__) number_of_queries = 'Number of Queries : {}'.format(end_queries - start_queries) time_taken = 'Finished in : {0:.2f}s'.format((end - start)) logger.error(function_name) logger.error(number_of_queries) logger.error(time_taken) return result
def handle_noargs(self, **options): data_reader = csv.reader(open(SOURCE_FILE)) i=0 for row in data_reader: if (i<2): fields = row else: j=0 row_dict = {} for column in fields: row_dict[column] = row[j] j = j + 1 db_row = FIPSCountyCongressDistrict(state_code=row_dict['STATE'], county_code=row_dict['COUNTY'], district_code=row_dict[CONGRESS_COLUMN], congress=CONGRESS) db_row.save() db.reset_queries() i = i + 1
def handle(self, *args, **options): self.encoding=options["encoding"] self.pages=os.path.join(args[0],"pages") transaction.enter_transaction_management() self.user=User.objects.order_by("id")[0] # Get first created user as owner self.language=Language.objects.get(name=options["language"]) # Find category self.tags=options["tags"] oc=len(gc.get_objects()) for page in os.listdir(self.pages): self.convert_page(page) reset_queries() gc.collect() new_oc=len(gc.get_objects()) self.out("%d leaked objects\n"%(new_oc-oc)) oc=new_oc transaction.commit() transaction.leave_transaction_management()
def __exit__(self, exc_type, exc_val, exc_tb): self.time_total = default_timer() - self.start_time if self.db_profile: settings.DEBUG = self.old_debug self.queries = connection.queries self.num_tx = len( [q for q in self.queries if q['sql'].startswith('SAVEPOINT')]) reset_queries() # assert number of queries if specified if self.assert_queries is not None: self.test.assertEqual(len(self.queries), self.assert_queries) # assert number of transactions if specified if self.assert_tx is not None: self.test.assertEqual(self.num_tx, self.assert_tx)
def handle(self, *prefix, **options): if not prefix: print(self.help) else: count = updated = 0 total = Media.objects.count() pb = ProgressBar(total) for i in range(0, total, 100): for media in Media.objects.all()[i:i + 100]: if media.url.startswith(prefix): media.url = media.url[len(prefix):] media.save() updated += 1 count += 1 pb.update(count) reset_queries() pb.done() print("Updated %d/%d media objects" % (updated, count))
def read_tree_time(model): # т.к. используется "ленивая" модель запросов, то чтобы запрос отправиля в базу # он преобразуется в список if model.__name__ == "Raw": list(model.objects.all().order_by("parent_id")) elif model.__name__ == "Ltree": list(model.objects.all().order_by("path")) else: list(model.objects.all()) result = { MODEL_FIELD: model.__name__, OPERATION_FIELD: "read_tree", TIME_FIELD: float(connection.queries[0]["time"]) } reset_queries() return result
def test_uses_backend_specific_quoting(self): """Backend-specific quotes should be used Table and field names should be quoted with the quote_name function provided by the database backend. The test here is a bit trivial since a real-life test case with PostgreSQL schema tricks or other table/field name munging would be difficult. """ qn = db.connection.ops.quote_name qs = _select_related_instances(Entry, 'section', [1], 'batch_select_entry', 'section_id') db.reset_queries() list(qs) sql = db.connection.queries[-1]['sql'] self.failUnless( sql.startswith('SELECT (%s.%s) AS ' % (qn('batch_select_entry'), qn('section_id'))))
def test_batch_select_related_quoted_section_id(self): """Field names should be quoted in the WHERE clause PostgreSQL is particularly picky about quoting when table or field names contain mixed case """ section = Section.objects.create(name='s1') entry = Entry.objects.create(section=section) db.reset_queries() sections = Section.objects.batch_select('entry').all() sections[0] sql = db.connection.queries[-1]['sql'] correct_where = ' WHERE "batch_select_entry"."section_id" IN (1)' self.failUnless( sql.endswith(correct_where), '"section_id" is not correctly quoted in the WHERE ' 'clause of %r' % sql)
def handler(request, context): try: if (f"{self.service_class.__name__}.{action}" not in grpc_settings.IGNORE_LOG_FOR_ACTION): logger.info( f"Receive action {action} on service {self.service_class.__name__}" ) # db connection state managed similarly to the wsgi handler db.reset_queries() # INFO - AM - 30/06/2021 - Need this in production environnement to avoid SSL end of files errors when too much connection on database close_old_connections() service_instance = self.create_service() service_instance.request = request service_instance.context = GRPCSocioProxyContext( context, action) service_instance.action = action service_instance.before_action() # INFO - AM - 05/05/2021 - getting the real function in the service and then calling it if necessary instance_action = getattr(service_instance, action) if asyncio.iscoroutinefunction(instance_action): instance_action = async_to_sync(instance_action) return instance_action(service_instance.request, service_instance.context) except GRPCException as grpc_error: logger.error(grpc_error) context.abort(grpc_error.status_code, grpc_error.get_full_details()) except Exception as error: etype, value, tb = sys.exc_info() formatted_exception = traceback.format_exception( etype, value, tb) # No need to send it to µservices logging because we did it as exception with log_unhandled_exception logger.error("".join(formatted_exception), extra={"emit_to_server": False}) grpcHandler = GRPCHandler() grpcHandler.log_unhandled_exception(etype, value, tb) context.abort(grpc.StatusCode.UNKNOWN, str(error)) finally: # INFO - AM - 30/06/2021 - Need this in production environnement to avoid SSL end of files errors when too much connection on database close_old_connections() pass
def index_pages(): """index all the pages that are modeled in the database """ solr = SolrConnection(settings.SOLR) solr.delete_query('type:page') cursor = connection.cursor() cursor.execute("SELECT id FROM core_page") count = 0 while True: row = cursor.fetchone() if row is None: break page = models.Page.objects.get(id=row[0]) LOGGER.info("[%s] indexing page: %s", count, page.url) solr.add(**page.solr_doc) count += 1 if count % 100 == 0: reset_queries() solr.commit()
def wrapper(*args, **kwargs): reset_queries() number_of_start_queries = len(connection.queries) start = time.perf_counter() result = func(*args, **kwargs) end = time.perf_counter() number_of_end_queries = len(connection.queries) print( f"-------------------------------------------------------------------" ) print(f"Function : {func.__name__}") print( f"Number of Queries : {number_of_end_queries-number_of_start_queries}" ) print(f"Finished in : {(end - start):.2f}s") print( f"-------------------------------------------------------------------" ) return result
def create_initial_revisions(self, model_class, comment, batch_size, verbosity, revision_manager, database): # Check all models for empty revisions. if verbosity >= 2: self.stdout.write("Creating initial revision(s) for model %s ..." % (force_text(model_class._meta.verbose_name))) created_count = 0 content_type = revision_manager._get_content_type(model_class, db=database) live_objs = model_class._base_manager.using(database).exclude( pk__reversion_in=(Version.objects.using(database).filter( content_type=content_type, ), "object_id")) # Save all the versions. ids = list(live_objs.values_list("pk", flat=True).order_by()) total = len(ids) for i in range(0, total, batch_size): chunked_ids = ids[i:i + batch_size] objects = live_objs.in_bulk(chunked_ids) for id, obj in objects.items(): try: revision_manager.save_revision( objects=(obj, ), comment=comment, db=database, ) except: self.stdout.write( "ERROR: Could not save initial version for %s %s." % ( model_class.__name__, obj.pk, )) raise created_count += 1 reset_queries() if verbosity >= 2: self.stdout.write("Created %s of %s." % (created_count, total)) # Print out a message, if feeling verbose. if verbosity >= 2: self.stdout.write("Created %s initial revision(s) for model %s." % ( created_count, force_text(model_class._meta.verbose_name), ))
def test_get_viewable_channel_query_performance(self): """ Test that we are not running too many queries in order to return a single viewable channel result. """ settings.DEBUG = True self.channel.viewers.add(self.user) self.channel.save() reset_queries() response = self.client.get(reverse("channel-list"), data={"view": True}) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 1) # This is a warning sign for performance problems, so if the number of queries goes above this # number, we need to evaluate the change and see if we can do something to optimize. self.assertQueriesLessThan(10)
def test_list_queries(self, client, expand): """Queries should be constant""" small_size = 1 users = UserFactory.create_batch(small_size) organization = OrganizationFactory(members=users) client.force_authenticate(user=users[0]) reset_queries() client.get(f"/api/users/?expand={expand}") num_queries = len(connection.queries) size = 10 users = UserFactory.create_batch(size) for user in users: Membership.objects.create(user=user, organization=organization) client.force_authenticate(user=users[0]) reset_queries() response = client.get(f"/api/users/?expand={expand}") assert num_queries == len(connection.queries) assert len(response.json()["results"]) == size + small_size