Example #1
0
    def check_gc_during_creation(self, makeref):
        thresholds = gc.get_threshold()
        gc.set_threshold(1, 1, 1)
        gc.collect()
        class A:
            pass

        def callback(*args):
            pass

        referenced = A()

        a = A()
        a.a = a
        a.wr = makeref(referenced)

        try:
            # now make sure the object and the ref get labeled as
            # cyclic trash:
            a = A()
            weakref.ref(referenced, callback)
            if test_support.due_to_ironpython_incompatibility():
                keepalive(referenced)

        finally:
            gc.set_threshold(*thresholds)
Example #2
0
def init(**kwargs):
	global screen, shoot, xFrame, yFrame
	os.environ['SDL_VIDEO_WINDOW_POS'] = "%d,%d" % (0,0)
	pygame.init()
	#sound staff
	pygame.mixer.pre_init(buffer=4096)
	pygame.mixer.init()
	shoot.append(pygame.mixer.Sound("gun1-1.wav"))
	pygame.mixer.set_num_channels(10)
	pygame.mixer.set_reserved(8)

	for i in range(0,7):
		channel.append(pygame.mixer.Channel(i))

	gc.enable()							#garbig kolektor
	gc.set_threshold(1,1,1)
	for i in range(0, kwargs["numb"]):
		rawClips.append(videoClip( folder=kwargs["folder"] , name = repr(i)))
	
	#displej stuff
	if kwargs["mod"] == "full":
		screen = pygame.display.set_mode(kwargs["resolution"],pygame.FULLSCREEN)
		xFrame = screen.get_width()
		yFrame = screen.get_height()

	else:
		screen = pygame.display.set_mode(kwargs["resolution"], pygame.NOFRAME)
		xFrame = kwargs["resolution"][0]
		yFrame = kwargs["resolution"][1]

	print xFrame, yFrame
	pygame.display.set_caption('KIA game')
	screen.fill((235,235,235))
    def handle(self, *args, **options):

        # Check that we can run
        if not settings.CENTRAL_SERVER:
            raise CommandError("This must only be run on the central server.")
        supported_langs = get_supported_languages()
        if not options["lang_codes"]:
            lang_codes = supported_langs
        else:
            requested_codes = set(options["lang_codes"].split(","))
            lang_codes = [lcode_to_ietf(lc) for lc in requested_codes if lc in supported_langs]
            unsupported_codes = requested_codes - set(lang_codes)
            if unsupported_codes:
                raise CommandError("Requested unsupported languages: %s" % sorted(list(unsupported_codes)))

        # Scrub options
        for key in options:
            # If no_update is set, then disable all update options.
            if key.startswith("update_"):
                options[key] = options[key] and not options["no_update"]

        if version_diff(options["version"], COMMAND_MIN_VERSION) < 0:
            raise CommandError("This command cannot be used for versions before %s" % COMMAND_MIN_VERSION)

        if options['low_mem']:
            logging.info('Making the GC more aggressive...')
            gc.set_threshold(36, 2, 2)

        # For dealing with central server changes across versions
        upgrade_old_schema()

        # Now, we're going to build the language packs, collecting metadata long the way.
        package_metadata = update_language_packs(lang_codes, options)
Example #4
0
    def check_gc_during_creation(self, makeref):
        # XXX: threshold not applicable to Jython
        if not test_support.is_jython:
            thresholds = gc.get_threshold()
            gc.set_threshold(1, 1, 1)
        gc.collect()
        class A:
            pass

        def callback(*args):
            pass

        referenced = A()

        a = A()
        a.a = a
        a.wr = makeref(referenced)

        try:
            # now make sure the object and the ref get labeled as
            # cyclic trash:
            a = A()
            weakref.ref(referenced, callback)

        finally:
            # XXX: threshold not applicable to Jython
            if not test_support.is_jython:
                gc.set_threshold(*thresholds)
Example #5
0
File: util.py Project: KDE/kajongg
def checkMemory():
    """as the name says"""
    # pylint: disable=too-many-branches
    if not Debug.gc:
        return
    gc.set_threshold(0)
    gc.set_debug(gc.DEBUG_LEAK)
    gc.enable()
    print('collecting {{{')
    gc.collect()        # we want to eliminate all output
    print('}}} done')

    # code like this may help to find specific things
    if True: # pylint: disable=using-constant-test
        interesting = ('Client', 'Player', 'Game')
        for obj in gc.garbage:
            if hasattr(obj, 'cell_contents'):
                obj = obj.cell_contents
            if not any(x in repr(obj) for x in interesting):
                continue
            for referrer in gc.get_referrers(obj):
                if referrer is gc.garbage:
                    continue
                if hasattr(referrer, 'cell_contents'):
                    referrer = referrer.cell_contents
                if referrer.__class__.__name__ in interesting:
                    for referent in gc.get_referents(referrer):
                        print('%s refers to %s' % (referrer, referent))
                else:
                    print('referrer of %s/%s is: id=%s type=%s %s' %
                          (type(obj), obj, id(referrer),
                           type(referrer), referrer))
    print('unreachable:%s' % gc.collect())
    gc.set_debug(0)
Example #6
0
    def check_gc_during_creation(self, makeref):
        if test_support.check_impl_detail():
            import gc
            thresholds = gc.get_threshold()
            gc.set_threshold(1, 1, 1)
        gc_collect()
        class A:
            pass

        def callback(*args):
            pass

        referenced = A()

        a = A()
        a.a = a
        a.wr = makeref(referenced)

        try:
            # now make sure the object and the ref get labeled as
            # cyclic trash:
            a = A()
            weakref.ref(referenced, callback)

        finally:
            if test_support.check_impl_detail():
                gc.set_threshold(*thresholds)
Example #7
0
 def setUp(self):
   TestCase.setUp(self)
   gc.set_threshold(500, 10, 10)
   self._started = False
   self._ioloop = rpclib.ioloop()
   rpclib.__IOLOOP__ = self._ioloop
   self._exc_info = None
Example #8
0
    def run():
        # make sure that we run the reactor with the sentinel log context,
        # otherwise other PreserveLoggingContext instances will get confused
        # and complain when they see the logcontext arbitrarily swapping
        # between the sentinel and `run` logcontexts.
        with PreserveLoggingContext():
            logger.info("Running")
            if cpu_affinity is not None:
                # Turn the bitmask into bits, reverse it so we go from 0 up
                mask_to_bits = bin(cpu_affinity)[2:][::-1]

                cpus = []
                cpu_num = 0

                for i in mask_to_bits:
                    if i == "1":
                        cpus.append(cpu_num)
                    cpu_num += 1

                p = psutil.Process()
                p.cpu_affinity(cpus)

            change_resource_limit(soft_file_limit)
            if gc_thresholds:
                gc.set_threshold(*gc_thresholds)
            reactor.run()
def main():
	arguments = parse_arguments()
	Monster.ATTRIBUTE_NAME_LIST = [
		"attribute_{0}".format(i) for i in xrange(arguments.attribute_count)]

	test_list = [
		("GC(threshold0:700)", 700, 0),
		("GC(threshold0:10000)", 10000, 0),
		("Manual", None, 1),
		("Weakref", None, 2),
	]

	for test_name, threshold, unloop_model in test_list:
		if threshold is not None:
			gc.set_threshold(threshold)
		run_test_and_plot(
			test_name,
			arguments.max_iteration,
			arguments.dungeon_per_iter,
			arguments.monster_per_dungeon,
			arguments.attribute_count,
			unloop_model)
	lgd = plt.legend(loc="center left", bbox_to_anchor=(1, 0.5))
	plt.savefig(
		arguments.plot_path, bbox_extra_artists=[lgd,], bbox_inches="tight")
Example #10
0
    def setUp (self):
        super (NotifyTestCase, self).setUp ()

        gc.set_threshold (0, 0, 0)

        self.__num_collectable_objects = self.collect_garbage ()
        self.__num_active_protections  = AbstractGCProtector.default.num_active_protections
Example #11
0
 def run():
     with LoggingContext("run"):
         logger.info("Running")
         change_resource_limit(config.soft_file_limit)
         if config.gc_thresholds:
             gc.set_threshold(*config.gc_thresholds)
         reactor.run()
Example #12
0
    def check_gc_during_creation(self, makeref):
        # gc.get/set_threshold does not exist e.g. in pypy
        thresholds = gc.get_threshold()
        gc.set_threshold(1, 1, 1)
        gc.collect()
        class A:
            pass

        def callback(*args):
            pass

        referenced = A()

        a = A()
        a.a = a
        a.wr = makeref(referenced)

        try:
            # now make sure the object and the ref get labeled as
            # cyclic trash:
            a = A()
            weakref.ref(referenced, callback)

        finally:
            gc.set_threshold(*thresholds)
Example #13
0
def gc_callback(option, opt, GC_THRESHOLD, *args):
    import gc
    if GC_THRESHOLD == 0:
        gc.disable()
        print "gc disabled"
    else:
        gc.set_threshold(GC_THRESHOLD)
        print "gc threshold:", gc.get_threshold()
Example #14
0
 def in_thread():
     # Uncomment to enable tracing of log context changes.
     # sys.settrace(logcontext_tracer)
     with LoggingContext("run"):
         change_resource_limit(hs.config.soft_file_limit)
         if hs.config.gc_thresholds:
             gc.set_threshold(*hs.config.gc_thresholds)
         reactor.run()
def tearDown(test):
    sys.path[:], sys.argv[:] = test.globs['saved-sys-info'][:2]
    if hasattr(gc, 'get_threshold'):
        gc.set_threshold(*test.globs['saved-gc-threshold'])
    sys.modules.clear()
    sys.modules.update(test.globs['saved-sys-info'][2])
    if sys.version_info[0] >= 3:
        doctest._SpoofOut = test.globs['saved-doctest-SpoofOut']
Example #16
0
def main():
	parser=argparse.ArgumentParser(description="access to pythons built-in garbage collector")
	parser.add_argument("command",help="what to do",choices=["enable","disable","status","collect","threshold","debug","break"],action="store")
	parser.add_argument("args",help="argument for command",action="store",nargs="*")
	ns=parser.parse_args()
	if ns.command=="enable":
		gc.enable()
	elif ns.command=="disable":
		gc.disable()
	elif ns.command=="collect":
		gc.collect()
	elif ns.command=="status":
		print "GC enabled:              {s}".format(s=gc.isenabled())
		tracked=gc.get_objects()
		n=len(tracked)
		print "Tracked objects:         {n}".format(n=n)
		size=sum([sys.getsizeof(e) for e in tracked])
		del tracked#this list may be big, better delete it
		print "Size of tracked objects: {s} bytes".format(s=size)
		print "Garbage:                 {n}".format(n=len(gc.garbage))
		gsize=sum([sys.getsizeof(e) for e in gc.garbage])
		print "Size of garbage:         {s} bytes".format(s=gsize)
		print "Debug:                   {d}".format(d=gc.get_debug())
	elif ns.command=="threshold":
		if len(ns.args)==0:
			print "Threshold:\n   G1: {}\n   G2: {}\n   G3: {}".format(*gc.get_threshold())
		elif len(ns.args)>3:
			print "Error: to many arguments for threshold!"
			sys.exit(1)
		else:
			try:
				ts=tuple([int(e) for e in ns.args])
			except ValueError:
				print "Error: expected arguments to be integer!"
				sys.exit(1)
			gc.set_threshold(*ts)
	elif ns.command=="debug":
		if len(ns.args)==0:
			print "Debug: {d}".format(d=gc.get_debug())
		elif len(ns.args)==1:
			try:
				flag=int(ns.args[0])
			except ValueError:
				print "Error: expected argument to be an integer!"
				sys.exit(1)
			gc.set_debug(flag)
		else:
			print "Error: expected exactly one argument for threshold!"
			sys.exit(1)
	elif ns.command=="break":
		if len(gc.garbage)==0:
			print "Error: No Garbage found!"
			sys.exit(1)
		else:
			for k in dir(garbage[0]):
				try: delattr(garbage,k)
				except: pass
			del gc.garbage[:]
Example #17
0
 def tearDown(self):
     gc.collect()
     self.assertEqual(len(gc.garbage), self.gc_count)
     if len(gc.garbage)>0:
         if self.verbose>1: print gc.get_objects()
         #TODO be pedantic and fail?
     del gc.garbage[:]
     gc.set_threshold(*self.gc_threshold_old)
     gc.set_debug(self.gc_flags_old)
Example #18
0
    def setUp (self):
        super (NotifyTestCase, self).setUp ()

        if not self.HAVE_CONTROLLABLE_GC:
            return

        gc.set_threshold (0, 0, 0)

        self.__num_collectable_objects = self.collect_garbage ()
        self.__num_active_protections = AbstractGCProtector.default.num_active_protections
Example #19
0
	def set_gc_threshold(self, iter1, iter2, iter3):
		"""
		Unknown

		@param
		@type

		@return: Unknown
		@rtype: Unknown
		"""
		gc.set_threshold(iter1, iter2, iter3)
Example #20
0
def main():
	"""
	The initial setup for blamehangle. Parse command line options, read config
	file, and start the Postman.
	"""
	
	#gc.set_debug(gc.DEBUG_STATS|gc.DEBUG_COLLECTABLE|gc.DEBUG_UNCOLLECTABLE|gc.DEBUG_INSTANCES|gc.DEBUG_OBJECTS)
	# set our own gc thresholds, to keep mem usage from creeping. It seems
	# that the default is extremely slow, and since blamehangle is not an
	# interactive program the minor performance hit taken during garbage
	# collection won't even be noticable.
	#
	# The default is (700, 10, 10) !
	gc.set_threshold(50, 5, 2)
	
	# Parse our command line options
	try:
		opts, args = getopt.getopt(sys.argv[1:], 'c:p', [ 'config=', 'profile' ])
	except getopt.GetoptError:
		Show_Usage()
	
	ConfigFile = None
	Profiled = 0
	for opt, arg in opts:
		if opt in ('-c', '--config'):
			ConfigFile = arg
		if opt in ('-p', '--profile'):
			Profiled = 1
	
	# Load our config file
	if not ConfigFile or not os.path.exists(ConfigFile):
		Show_Usage()
	
	Config = ConfigParser()
	Config.read(ConfigFile)
	
	# Start up the Postman, and run him forever. If we're profiling, do that.
	Post = Postman(ConfigFile, Config)
	
	if Profiled:
		import hotshot
		prof = hotshot.Profile('hangle.prof')
		prof.runcall(Post.run_forever)
		prof.close()
		
		# Print some profile stats
		import hotshot.stats
		stats = hotshot.stats.load('hangle.prof')
		stats.strip_dirs()
		stats.sort_stats('time', 'calls')
		stats.print_stats(25)
	
	else:
		Post.run_forever()
Example #21
0
    def global_setup(self):
        self.old_threshold = gc.get_threshold()

        if self.threshold[0]:
            self.runner.options.output.info(
                "Cyclic garbage collection threshold set to: %s" %
                repr(tuple(self.threshold)))
        else:
            self.runner.options.output.info(
                "Cyclic garbage collection is disabled.")

        gc.set_threshold(*self.threshold)
Example #22
0
def doctestsTearDown(test):
    sys.path[:], sys.argv[:] = test.globs['saved-sys-info'][:2]
    gc.set_threshold(*test.globs['saved-sys-info'][3])
    sys.modules.clear()
    sys.modules.update(test.globs['saved-sys-info'][2])
    try:
        testrunner.configure_logging = test.globs['old_configure_logging']
        del test.globs['old_configure_logging']
    except KeyError:
        pass
    del os.environ[z3c.etree.testing.engine_env_key]
    del test.globs['old_engine']
Example #23
0
def Main():
  """Parse command line, fork, and start stream request handler."""
  # Remember the time spent in the parent.
  times_at_start = os.times()
  include_server_port, pid_file = _ParseCommandLineOptions()
  # Get locking mechanism.
  include_server_port_ready = _IncludeServerPortReady()
  # Now spawn child so that parent can exit immediately after writing
  # the process id of child to the pid file.
  times_at_fork = os.times()
  pid = os.fork()
  if pid != 0: 
    # In parent.
    #
    if pid_file:
      pid_file_fd = open(pid_file, "w")
      print >> pid_file_fd, pid
      pid_file_fd.close()
    # Just run to completion now -- after making sure that child is ready.
    include_server_port_ready.Acquire()
    # concerned.
  else:
    # In child.
    #
    # We call _Setup only now, because the process id, used in naming the client
    # root, must be that of this process, not that of the parent process. See
    # _CleanOutOthers for the importance of the process id.
    (include_analyzer, server) = _SetUp(include_server_port)
    include_server_port_ready.Release()
    try:
      try:
        gc.set_threshold(basics.GC_THRESHOLD)
        # Use commented-out line below to have a message printed for each
        # collection.
        # gc.set_debug(gc.DEBUG_STATS + gc.DEBUG_COLLECTABLE)
        server.serve_forever()
      except KeyboardInterrupt:
        print >> sys.stderr, (
            "Include server: keyboard interrupt, quitting after cleaning up.")
        _CleanOut(include_analyzer, include_server_port)
      except SignalSIGTERM:
        Debug(DEBUG_TRACE, "Include server shutting down.")
        _CleanOut(include_analyzer, include_server_port)
      except:
        print >> sys.stderr, (
            "Include server: exception occurred, quitting after cleaning up.")
        _PrintStackTrace(sys.stderr)
        _CleanOut(include_analyzer, include_server_port)
        raise # reraise exception
    finally:
      if basics.opt_print_times:
        _PrintTimes(times_at_start, times_at_fork, os.times())
Example #24
0
	def applySettings(self):
		# GC
		t = gc.get_threshold()
		multi = self.gcMemoryThreshold
		gc.set_threshold(int(t[0] * multi), int(t[1] * multi), int(t[2] * multi))
		
		# Fonts
		self.applyMonospaceFont(self.monospaceFont)
		self.applyStandardFont(self.standardFont)
		self.applyTheme(self.themeName)
		
		if os.name == "nt":
			self.applyMenuFont(self.standardFont)
 def _adjustGcThreshold(self, task):
     numGarbage = GarbageReport.checkForGarbageLeaks()
     if numGarbage == 0:
         self.gcNotify.debug('no garbage found, doubling gc threshold')
         a, b, c = gc.get_threshold()
         gc.set_threshold(min(a * 2, 1 << 30), b, c)
         task.delayTime = task.delayTime * 2
         retVal = Task.again
     else:
         self.gcNotify.warning('garbage found, reverting gc threshold')
         gc.set_threshold(*self._gcDefaultThreshold)
         retVal = Task.done
     return retVal
Example #26
0
def test_set_threshold():
    #the method has three arguments
    gc.set_threshold(0,-2,2)
    result = gc.get_threshold()
    AreEqual(result[0],0)
    AreEqual(result[1],-2)
    AreEqual(result[2],2)
    
    ##the method has two argument
    gc.set_threshold(0,128)
    result = gc.get_threshold()
    AreEqual(result[0],0)
    AreEqual(result[1],128)
    #CodePlex Work Item 8523
    #AreEqual(result[2],2)
    
   
    #the method has only one argument
    gc.set_threshold(-10009)
    result= gc.get_threshold()
    AreEqual(result[0],-10009)
    #CodePlex Work Item 8523
    #AreEqual(result[1],128)
    #AreEqual(result[2],2)
    
    #the argument is a random int
    for i in xrange(1,65535,6):
        gc.set_threshold(i)
        result = gc.get_threshold()
        AreEqual(result[0],i)
    def test_del_newclass(self):
        # __del__ methods can trigger collection, make this to happen
        thresholds = gc.get_threshold()
        gc.enable()
        gc.set_threshold(1)

        class A(object):
            def __del__(self):
                dir(self)
        a = A()
        del a

        gc.disable()
        gc.set_threshold(*thresholds)
Example #28
0
    def setUp(self):
        self.mem_ini = MemorySingleton(self.verbose - 1)
        self.mem_ref = MemoryStatistics(self.verbose - 1)
        self.mem_cur = self.mem_ref.copy()

        self.gc_threshold_old = gc.get_threshold()
        self.gc_flags_old = gc.get_debug()
        gc.set_threshold(*self.gc_threshold)
        gc.set_debug(self.gc_flags)

        # Try to obtain a clean slate
        gc.collect()
        self.gc_count = len(gc.garbage)
        del gc.garbage[:]
Example #29
0
def unreachable_items():
  # see http://stackoverflow.com/questions/16911559/trouble-understanding-pythons-gc-garbage-for-tracing-memory-leaks
  # first time setup
  import gc
  gc.set_threshold( 0 ) # only manual sweeps
  gc.set_debug( gc.DEBUG_SAVEALL ) # keep unreachable items as garbage
  gc.enable() # start gc if not yet running (is this necessary?)
  # operation
  if gc.collect() == 0:
    log.info( 'no unreachable items' )
  else:
    fmt = '[%%0%dd] %%s' % len(str(len(gc.garbage)-1))
    log.info( 'unreachable items:\n '
            + '\n '.join( fmt % item for item in enumerate( gc.garbage ) ) )
    _deep_purge_list( gc.garbage ) # remove unreachable items
Example #30
0
    def test_set_threshold(self):
        """get_threshold, set_threshold"""

        #the method has three arguments
        gc.set_threshold(0,-2,2)
        result = gc.get_threshold()
        self.assertEqual(result[0],0)
        self.assertEqual(result[1],-2)
        self.assertEqual(result[2],2)
        
        ##the method has two argument
        gc.set_threshold(0,128)
        result = gc.get_threshold()
        self.assertEqual(result[0],0)
        self.assertEqual(result[1],128)
        #CodePlex Work Item 8523
        self.assertEqual(result[2],2)
        
    
        #the method has only one argument
        gc.set_threshold(-10009)
        result= gc.get_threshold()
        self.assertEqual(result[0],-10009)
        #CodePlex Work Item 8523
        self.assertEqual(result[1],128)
        self.assertEqual(result[2],2)
        
        #the argument is a random int
        for i in xrange(1,65535,6):
            gc.set_threshold(i)
            result = gc.get_threshold()
            self.assertEqual(result[0],i)
        
        #a argument is a float
        #CodePlex Work Item 8522
        self.assertRaises(TypeError,gc.set_threshold,2.1)
        self.assertRaises(TypeError,gc.set_threshold,3,-1.3)
        
        #a argument is a string
        #CodePlex Work Item 8522
        self.assertRaises(TypeError,gc.set_threshold,"1")
        self.assertRaises(TypeError,gc.set_threshold,"str","xdv#4")
        self.assertRaises(TypeError,gc.set_threshold,2,"1")
        self.assertRaises(TypeError,gc.set_threshold,31,-123,"asdfasdf","1")
        
        #a argument is a object
        #CodePlex Work Item 8522
        o  = object()
        o2 = object()
        self.assertRaises(TypeError,gc.set_threshold,o)
        self.assertRaises(TypeError,gc.set_threshold,o,o2)
        self.assertRaises(TypeError,gc.set_threshold,1,-123,o)
        o  = _random.Random()
        o2 = _random.Random()
        self.assertRaises(TypeError,gc.set_threshold,o)
        self.assertRaises(TypeError,gc.set_threshold,o,o2)
        self.assertRaises(TypeError,gc.set_threshold,8,64,o)
Example #31
0
def main():

    log_clock("Bootstrap to the start of init.init")

    renpy.game.exception_info = 'Before loading the script.'

    # Get ready to accept new arguments.
    renpy.arguments.pre_init()

    # Init the screen language parser.
    renpy.sl2.slparser.init()

    # Init the config after load.
    renpy.config.init()

    # Set up variants.
    choose_variants()
    renpy.display.touch = "touch" in renpy.config.variants

    log_clock("Early init")

    # Note the game directory.
    game.basepath = renpy.config.gamedir
    renpy.config.searchpath = [renpy.config.gamedir]

    # Find the common directory.
    commondir = __main__.path_to_common(
        renpy.config.renpy_base)  # E1101 @UndefinedVariable

    if os.path.isdir(commondir):
        renpy.config.searchpath.append(commondir)
        renpy.config.commondir = commondir
    else:
        renpy.config.commondir = None

    # Add path from env variable, if any
    if "RENPY_SEARCHPATH" in os.environ:
        renpy.config.searchpath.extend(
            os.environ["RENPY_SEARCHPATH"].split("::"))

    if renpy.android:
        renpy.config.searchpath = []
        renpy.config.commondir = None

        if "ANDROID_PUBLIC" in os.environ:
            android_game = os.path.join(os.environ["ANDROID_PUBLIC"], "game")

            print("Android searchpath: ", android_game)

            if os.path.exists(android_game):
                renpy.config.searchpath.insert(0, android_game)

    # Load Ren'Py extensions.
    for dir in renpy.config.searchpath:  # @ReservedAssignment
        for fn in os.listdir(dir):
            if fn.lower().endswith(".rpe"):
                load_rpe(dir + "/" + fn)

    # The basename is the final component of the path to the gamedir.
    for i in sorted(os.listdir(renpy.config.gamedir)):

        if not i.endswith(".rpa"):
            continue

        i = i[:-4]
        renpy.config.archives.append(i)

    renpy.config.archives.reverse()

    # Initialize archives.
    renpy.loader.index_archives()

    # Start auto-loading.
    renpy.loader.auto_init()

    log_clock("Loader init")

    # Initialize the log.
    game.log = renpy.python.RollbackLog()

    # Initialize the store.
    renpy.store.store = sys.modules['store']

    # Set up styles.
    game.style = renpy.style.StyleManager()  # @UndefinedVariable
    renpy.store.style = game.style

    # Run init code in its own context. (Don't log.)
    game.contexts = [renpy.execution.Context(False)]
    game.contexts[0].init_phase = True

    renpy.execution.not_infinite_loop(60)

    # Load the script.
    renpy.game.exception_info = 'While loading the script.'
    renpy.game.script = renpy.script.Script()

    if renpy.session.get("compile", False):
        renpy.game.args.compile = True

    # Set up error handling.
    renpy.exports.load_module("_errorhandling")

    if renpy.exports.loadable("tl/None/common.rpym") or renpy.exports.loadable(
            "tl/None/common.rpymc"):
        renpy.exports.load_module("tl/None/common")

    renpy.config.init_system_styles()
    renpy.style.build_styles()  # @UndefinedVariable

    log_clock("Loading error handling")

    # If recompiling everything, remove orphan .rpyc files.
    # Otherwise, will fail in case orphan .rpyc have same
    # labels as in other scripts (usually happens on script rename).
    if (renpy.game.args.command == 'compile'
        ) and not (renpy.game.args.keep_orphan_rpyc):  # @UndefinedVariable

        for (fn, dn) in renpy.game.script.script_files:

            if dn is None:
                continue

            if not os.path.isfile(os.path.join(dn, fn + ".rpy")):

                try:
                    name = os.path.join(dn, fn + ".rpyc")
                    os.rename(name, name + ".bak")
                except OSError:
                    # This perhaps shouldn't happen since either .rpy or .rpyc should exist
                    pass

        # Update script files list, so that it doesn't contain removed .rpyc's
        renpy.loader.cleardirfiles()
        renpy.game.script.scan_script_files()

    # Load all .rpy files.
    renpy.game.script.load_script()  # sets renpy.game.script.
    log_clock("Loading script")

    if renpy.game.args.command == 'load-test':  # @UndefinedVariable
        start = time.time()

        for i in range(5):
            print(i)
            renpy.game.script = renpy.script.Script()
            renpy.game.script.load_script()

        print(time.time() - start)
        sys.exit(0)

    renpy.game.exception_info = 'After loading the script.'

    # Find the save directory.
    if renpy.config.savedir is None:
        renpy.config.savedir = __main__.path_to_saves(
            renpy.config.gamedir)  # E1101 @UndefinedVariable

    if renpy.game.args.savedir:  # @UndefinedVariable
        renpy.config.savedir = renpy.game.args.savedir  # @UndefinedVariable

    # Init preferences.
    game.persistent = renpy.persistent.init()
    game.preferences = game.persistent._preferences

    for i in renpy.game.persistent._seen_translates:  # @UndefinedVariable
        if i in renpy.game.script.translator.default_translates:
            renpy.game.seen_translates_count += 1

    if game.persistent._virtual_size:
        renpy.config.screen_width, renpy.config.screen_height = game.persistent._virtual_size

    # Init save locations and loadsave.
    renpy.savelocation.init()

    # We need to be 100% sure we kill the savelocation thread.
    try:

        # Init save slots.
        renpy.loadsave.init()

        log_clock("Loading save slot metadata.")

        # Load persistent data from all save locations.
        renpy.persistent.update()
        game.preferences = game.persistent._preferences
        log_clock("Loading persistent")

        # Clear the list of seen statements in this game.
        game.seen_session = {}

        # Initialize persistent variables.
        renpy.store.persistent = game.persistent
        renpy.store._preferences = game.preferences
        renpy.store._test = renpy.test.testast._test

        if renpy.parser.report_parse_errors():
            raise renpy.game.ParseErrorException()

        renpy.game.exception_info = 'While executing init code:'

        for _prio, node in game.script.initcode:

            if isinstance(node, renpy.ast.Node):
                node_start = time.time()

                renpy.game.context().run(node)

                node_duration = time.time() - node_start

                if node_duration > renpy.config.profile_init:
                    renpy.display.log.write(" - Init at %s:%d took %.5f s.",
                                            node.filename, node.linenumber,
                                            node_duration)

            else:
                # An init function.
                node()

        renpy.game.exception_info = 'After initialization, but before game start.'

        # Check if we should simulate android.
        renpy.android = renpy.android or renpy.config.simulate_android  # @UndefinedVariable

        # Re-set up the logging.
        renpy.log.post_init()

        # Run the post init code, if any.
        for i in renpy.game.post_init:
            i()

        renpy.game.script.report_duplicate_labels()

        # Sort the images.
        renpy.display.image.image_names.sort()

        game.persistent._virtual_size = renpy.config.screen_width, renpy.config.screen_height

        log_clock("Running init code")

        renpy.pyanalysis.load_cache()
        log_clock("Loading analysis data")

        # Analyze the script and compile ATL.
        renpy.game.script.analyze()
        renpy.atl.compile_all()
        log_clock("Analyze and compile ATL")

        # Index the archive files. We should not have loaded an image
        # before this point. (As pygame will not have been initialized.)
        # We need to do this again because the list of known archives
        # may have changed.
        renpy.loader.index_archives()
        log_clock("Index archives")

        # Check some environment variables.
        renpy.game.less_memory = "RENPY_LESS_MEMORY" in os.environ
        renpy.game.less_mouse = "RENPY_LESS_MOUSE" in os.environ
        renpy.game.less_updates = "RENPY_LESS_UPDATES" in os.environ

        renpy.dump.dump(False)
        renpy.game.script.make_backups()
        log_clock("Dump and make backups.")

        # Initialize image cache.
        renpy.display.im.cache.init()
        log_clock("Cleaning cache")

        # Make a clean copy of the store.
        renpy.python.make_clean_stores()
        log_clock("Making clean stores")

        gc.collect()

        if renpy.config.manage_gc:
            gc.set_threshold(*renpy.config.gc_thresholds)

            gc_debug = int(os.environ.get("RENPY_GC_DEBUG", 0))

            if renpy.config.gc_print_unreachable:
                gc_debug |= gc.DEBUG_SAVEALL

            gc.set_debug(gc_debug)

        log_clock("Initial gc.")

        # Start debugging file opens.
        renpy.debug.init_main_thread_open()

        # (Perhaps) Initialize graphics.
        if not game.interface:
            renpy.display.core.Interface()
            log_clock("Creating interface object")

        # Start things running.
        restart = None

        while True:

            if restart:
                renpy.display.screen.before_restart()

            try:
                try:
                    run(restart)
                finally:
                    restart = (renpy.config.end_game_transition,
                               "_invoke_main_menu", "_main_menu")
                    renpy.persistent.update(True)

            except game.FullRestartException as e:
                restart = e.reason

            finally:

                # Flush any pending interface work.
                renpy.display.interface.finish_pending()

                # Give Ren'Py a couple of seconds to finish saving.
                renpy.loadsave.autosave_not_running.wait(3.0)

    finally:

        gc.set_debug(0)

        renpy.loader.auto_quit()
        renpy.savelocation.quit()
        renpy.translation.write_updated_strings()

    # This is stuff we do on a normal, non-error return.
    if not renpy.display.error.error_handled:
        renpy.display.render.check_at_shutdown()
            if count[r] > 0:
                break
            r += 1
        else:
            return max_flips




if __name__ == "__main__":
    
    imp = platform.python_implementation()
    # gc.disable()

    if imp == "PyPy":
        hooks = MyHooks()
        gc.hooks.set(hooks)
    elif imp == "CPython":
        gc.set_threshold(1,2,2)
        gc.set_debug(gc.DEBUG_STATS)
    arg = DEFAULT_ARG
    fannkuch(arg)
    
    

    if imp == "PyPy":
        #print(gc.get_stats())
        pass    
    elif imp == "CPython":
        gc.set_debug(0)
Example #33
0
def main(
    host,
    port,
    bokeh_port,
    show,
    dashboard,
    bokeh,
    dashboard_prefix,
    use_xheaders,
    pid_file,
    scheduler_file,
    interface,
    protocol,
    local_directory,
    preload,
    preload_argv,
    tls_ca_file,
    tls_cert,
    tls_key,
    dashboard_address,
    idle_timeout,
):
    g0, g1, g2 = gc.get_threshold()  # https://github.com/dask/distributed/issues/1653
    gc.set_threshold(g0 * 3, g1 * 3, g2 * 3)

    enable_proctitle_on_current()
    enable_proctitle_on_children()

    if bokeh_port is not None:
        warnings.warn(
            "The --bokeh-port flag has been renamed to --dashboard-address. "
            "Consider adding ``--dashboard-address :%d`` " % bokeh_port
        )
        dashboard_address = bokeh_port
    if bokeh is not None:
        warnings.warn(
            "The --bokeh/--no-bokeh flag has been renamed to --dashboard/--no-dashboard. "
        )
        dashboard = bokeh

    if port is None and (not host or not re.search(r":\d", host)):
        port = 8786

    sec = Security(
        **{
            k: v
            for k, v in [
                ("tls_ca_file", tls_ca_file),
                ("tls_scheduler_cert", tls_cert),
                ("tls_scheduler_key", tls_key),
            ]
            if v is not None
        }
    )

    if not host and (tls_ca_file or tls_cert or tls_key):
        host = "tls://"

    if pid_file:
        with open(pid_file, "w") as f:
            f.write(str(os.getpid()))

        def del_pid_file():
            if os.path.exists(pid_file):
                os.remove(pid_file)

        atexit.register(del_pid_file)

    local_directory_created = False
    if local_directory:
        if not os.path.exists(local_directory):
            os.mkdir(local_directory)
            local_directory_created = True
    else:
        local_directory = tempfile.mkdtemp(prefix="scheduler-")
        local_directory_created = True
    if local_directory not in sys.path:
        sys.path.insert(0, local_directory)

    if sys.platform.startswith("linux"):
        import resource  # module fails importing on Windows

        soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
        limit = max(soft, hard // 2)
        resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))

    loop = IOLoop.current()
    logger.info("-" * 47)

    scheduler = Scheduler(
        loop=loop,
        scheduler_file=scheduler_file,
        security=sec,
        host=host,
        port=port,
        interface=interface,
        protocol=protocol,
        dashboard_address=dashboard_address if dashboard else None,
        service_kwargs={"dashboard": {"prefix": dashboard_prefix}},
        idle_timeout=idle_timeout,
        preload=preload,
        preload_argv=preload_argv,
    )
    logger.info("Local Directory: %26s", local_directory)
    logger.info("-" * 47)

    install_signal_handlers(loop)

    async def run():
        await scheduler
        await scheduler.finished()

    try:
        loop.run_sync(run)
    finally:
        scheduler.stop()
        if local_directory_created:
            shutil.rmtree(local_directory)

        logger.info("End scheduler at %r", scheduler.address)
Example #34
0
     generate = True
 elif o in ('-x', '--exclude'):
     exclude = True
 elif o in ('-s', '--single'):
     single = True
 elif o in ('-r', '--randomize'):
     randomize = True
 elif o in ('-f', '--fromfile'):
     fromfile = a
 elif o in ('-l', '--findleaks'):
     findleaks = True
 elif o in ('-L', '--runleaks'):
     runleaks = True
 elif o in ('-t', '--threshold'):
     import gc
     gc.set_threshold(int(a))
 elif o in ('-T', '--coverage'):
     trace = True
 elif o in ('-D', '--coverdir'):
     coverdir = os.path.join(os.getcwd(), a)
 elif o in ('-N', '--nocoverdir'):
     coverdir = None
 elif o in ('-R', '--huntrleaks'):
     huntrleaks = a.split(':')
     if len(huntrleaks) != 3:
         print a, huntrleaks
         usage(2, '-R takes three colon-separated arguments')
     if len(huntrleaks[0]) == 0:
         huntrleaks[0] = 5
     else:
         huntrleaks[0] = int(huntrleaks[0])
Example #35
0
def setup_tests(ns):
    try:
        stderr_fd = sys.__stderr__.fileno()
    except (ValueError, AttributeError):
        # Catch ValueError to catch io.UnsupportedOperation on TextIOBase
        # and ValueError on a closed stream.
        #
        # Catch AttributeError for stderr being None.
        stderr_fd = None
    else:
        # Display the Python traceback on fatal errors (e.g. segfault)
        faulthandler.enable(all_threads=True, file=stderr_fd)

        # Display the Python traceback on SIGALRM or SIGUSR1 signal
        signals = []
        if hasattr(signal, 'SIGALRM'):
            signals.append(signal.SIGALRM)
        if hasattr(signal, 'SIGUSR1'):
            signals.append(signal.SIGUSR1)
        for signum in signals:
            faulthandler.register(signum, chain=True, file=stderr_fd)

    _adjust_resource_limits()
    replace_stdout()
    support.record_original_stdout(sys.stdout)

    if ns.testdir:
        # Prepend test directory to sys.path, so runtest() will be able
        # to locate tests
        sys.path.insert(0, os.path.abspath(ns.testdir))

    # Some times __path__ and __file__ are not absolute (e.g. while running from
    # Lib/) and, if we change the CWD to run the tests in a temporary dir, some
    # imports might fail.  This affects only the modules imported before os.chdir().
    # These modules are searched first in sys.path[0] (so '' -- the CWD) and if
    # they are found in the CWD their __file__ and __path__ will be relative (this
    # happens before the chdir).  All the modules imported after the chdir, are
    # not found in the CWD, and since the other paths in sys.path[1:] are absolute
    # (site.py absolutize them), the __file__ and __path__ will be absolute too.
    # Therefore it is necessary to absolutize manually the __file__ and __path__ of
    # the packages to prevent later imports to fail when the CWD is different.
    for module in sys.modules.values():
        if hasattr(module, '__path__'):
            for index, path in enumerate(module.__path__):
                module.__path__[index] = os.path.abspath(path)
        if getattr(module, '__file__', None):
            module.__file__ = os.path.abspath(module.__file__)

    if ns.huntrleaks:
        unittest.BaseTestSuite._cleanup = False

    if ns.memlimit is not None:
        support.set_memlimit(ns.memlimit)

    if ns.threshold is not None:
        gc.set_threshold(ns.threshold)

    support.suppress_msvcrt_asserts(ns.verbose and ns.verbose >= 2)

    support.use_resources = ns.use_resources

    if hasattr(sys, 'addaudithook'):
        # Add an auditing hook for all tests to ensure PySys_Audit is tested
        def _test_audit_hook(name, args):
            pass
        sys.addaudithook(_test_audit_hook)

    setup_unraisable_hook()
    setup_threading_excepthook()

    if ns.timeout is not None:
        # For a slow buildbot worker, increase SHORT_TIMEOUT and LONG_TIMEOUT
        support.SHORT_TIMEOUT = max(support.SHORT_TIMEOUT, ns.timeout / 40)
        support.LONG_TIMEOUT = max(support.LONG_TIMEOUT, ns.timeout / 4)

        # If --timeout is short: reduce timeouts
        support.LOOPBACK_TIMEOUT = min(support.LOOPBACK_TIMEOUT, ns.timeout)
        support.INTERNET_TIMEOUT = min(support.INTERNET_TIMEOUT, ns.timeout)
        support.SHORT_TIMEOUT = min(support.SHORT_TIMEOUT, ns.timeout)
        support.LONG_TIMEOUT = min(support.LONG_TIMEOUT, ns.timeout)

    if ns.xmlpath:
        from test.support.testresult import RegressionTestResult
        RegressionTestResult.USE_XML = True

    # Ensure there's a non-ASCII character in env vars at all times to force
    # tests consider this case. See BPO-44647 for details.
    os.environ.setdefault(
        UNICODE_GUARD_ENV,
        "\N{SMILING FACE WITH SUNGLASSES}",
    )
Example #36
0
def main(logger):
    opt = parse_args()
    print(opt)

    # Garbage collection, default threshold is (700, 10, 10).
    # Set threshold lower to collect garbage more frequently and release more CPU memory for heavy data loading.
    gc.set_threshold(100, 5, 5)

    # set env
    num_gpus = opt.num_gpus
    batch_size = opt.batch_size
    context = [mx.cpu()]
    if num_gpus > 0:
        batch_size *= max(1, num_gpus)
        context = [mx.gpu(i) for i in range(num_gpus)]

    num_workers = opt.num_workers
    print('Total batch size is set to %d on %d GPUs' % (batch_size, num_gpus))

    # get data
    image_norm_mean = [0.485, 0.456, 0.406]
    image_norm_std = [0.229, 0.224, 0.225]
    if opt.ten_crop:
        transform_test = transforms.Compose([
            video.VideoTenCrop(opt.input_size),
            video.VideoToTensor(),
            video.VideoNormalize(image_norm_mean, image_norm_std)
        ])
        opt.num_crop = 10
    elif opt.three_crop:
        transform_test = transforms.Compose([
            video.VideoThreeCrop(opt.input_size),
            video.VideoToTensor(),
            video.VideoNormalize(image_norm_mean, image_norm_std)
        ])
        opt.num_crop = 3
    else:
        transform_test = video.VideoGroupValTransform(size=opt.input_size,
                                                      mean=image_norm_mean,
                                                      std=image_norm_std)
        opt.num_crop = 1

    if not opt.deploy:
        # get model
        if opt.use_pretrained and len(opt.hashtag) > 0:
            opt.use_pretrained = opt.hashtag
        classes = opt.num_classes
        model_name = opt.model
        # Currently, these is no hashtag for int8 models.
        if opt.quantized:
            model_name += '_int8'
            opt.use_pretrained = True

        net = get_model(name=model_name,
                        nclass=classes,
                        pretrained=opt.use_pretrained,
                        num_segments=opt.num_segments,
                        num_crop=opt.num_crop)
        net.cast(opt.dtype)
        net.collect_params().reset_ctx(context)
        if opt.mode == 'hybrid':
            net.hybridize(static_alloc=True, static_shape=True)
        if opt.resume_params is not '' and not opt.use_pretrained:
            net.load_parameters(opt.resume_params, ctx=context)
            print('Pre-trained model %s is successfully loaded.' %
                  (opt.resume_params))
        else:
            print(
                'Pre-trained model is successfully loaded from the model zoo.')
    else:
        model_name = 'deploy'
        net = mx.gluon.SymbolBlock.imports(
            '{}-symbol.json'.format(opt.model_prefix), ['data'],
            '{}-0000.params'.format(opt.model_prefix))
        net.hybridize(static_alloc=True, static_shape=True)

    print("Successfully loaded model {}".format(model_name))
    # dummy data for benchmarking performance
    if opt.benchmark:
        benchmarking(opt, net, context)
        sys.exit()

    if opt.dataset == 'ucf101':
        val_dataset = UCF101(setting=opt.val_list,
                             root=opt.data_dir,
                             train=False,
                             new_width=opt.new_width,
                             new_height=opt.new_height,
                             new_length=opt.new_length,
                             target_width=opt.input_size,
                             target_height=opt.input_size,
                             test_mode=True,
                             num_segments=opt.num_segments,
                             transform=transform_test)
    elif opt.dataset == 'kinetics400':
        val_dataset = Kinetics400(
            setting=opt.val_list,
            root=opt.data_dir,
            train=False,
            new_width=opt.new_width,
            new_height=opt.new_height,
            new_length=opt.new_length,
            new_step=opt.new_step,
            target_width=opt.input_size,
            target_height=opt.input_size,
            video_loader=opt.video_loader,
            use_decord=opt.use_decord,
            slowfast=opt.slowfast,
            slow_temporal_stride=opt.slow_temporal_stride,
            fast_temporal_stride=opt.fast_temporal_stride,
            test_mode=True,
            num_segments=opt.num_segments,
            num_crop=opt.num_crop,
            transform=transform_test)
    elif opt.dataset == 'somethingsomethingv2':
        val_dataset = SomethingSomethingV2(setting=opt.val_list,
                                           root=opt.data_dir,
                                           train=False,
                                           new_width=opt.new_width,
                                           new_height=opt.new_height,
                                           new_length=opt.new_length,
                                           new_step=opt.new_step,
                                           target_width=opt.input_size,
                                           target_height=opt.input_size,
                                           video_loader=opt.video_loader,
                                           use_decord=opt.use_decord,
                                           num_segments=opt.num_segments,
                                           transform=transform_test)
    elif opt.dataset == 'hmdb51':
        val_dataset = HMDB51(setting=opt.val_list,
                             root=opt.data_dir,
                             train=False,
                             new_width=opt.new_width,
                             new_height=opt.new_height,
                             new_length=opt.new_length,
                             new_step=opt.new_step,
                             target_width=opt.input_size,
                             target_height=opt.input_size,
                             video_loader=opt.video_loader,
                             use_decord=opt.use_decord,
                             num_segments=opt.num_segments,
                             transform=transform_test)
    else:
        logger.info('Dataset %s is not supported yet.' % (opt.dataset))

    val_data = gluon.data.DataLoader(val_dataset,
                                     batch_size=batch_size,
                                     shuffle=False,
                                     num_workers=num_workers,
                                     prefetch=int(opt.prefetch_ratio *
                                                  num_workers),
                                     last_batch='discard')
    print('Load %d test samples in %d iterations.' %
          (len(val_dataset), len(val_data)))

    # calibrate FP32 model into INT8 model
    if opt.calibration:
        calibration(net, val_data, opt, context, logger)
        sys.exit()

    start_time = time.time()
    acc_top1_val, acc_top5_val = test(context, val_data, opt, net)
    end_time = time.time()

    print('Test accuracy: acc-top1=%f acc-top5=%f' %
          (acc_top1_val * 100, acc_top5_val * 100))
    print('Total evaluation time is %4.2f minutes' %
          ((end_time - start_time) / 60))
Example #37
0
import csv
import gc
start=time.time()
csvpd=pd.DataFrame(columns=['date','time','close','volume'])
global microsec
microsec=0.0000
global lasttime
lasttime=''
with open('data.csv',mode='r',encoding='utf-8',newline='') as file:
    rows=csv.reader(file)
    for row in rows:
        if row[0].strip() !='\ufeff成交日期' and row[1].strip()=='TX' and row[2].strip()=='201809':
            #成交日期,商品代號,到期月份(週別),成交時間,成交價格,成交數量(B+S),近月價格,遠月價格,開盤集合競價 
            row[0]=datetime.datetime.strptime(str(row[0]).strip(),'%Y%m%d').strftime('%Y/%m/%d')
            if row[3].strip() != lasttime :
                lasttime=row[3].strip()
                microsec=0.0000
                row[3]=datetime.datetime.strptime(row[3].strip(),'%H%M%S').strftime('%H:%M:%S')
            else:
                microsec+=0.0001
                row[3]=datetime.datetime.strptime(row[3].strip(),'%H%M%S').strftime('%H:%M:%S')+str(microsec)[1:6]
            #print(type(row[0]),type(row[3]),type(row[4]),type(row[5]))
            newlist=[[row[0],row[3],row[4].strip(),row[5].strip()]]
            csvpd=csvpd.append(pd.DataFrame(newlist,columns=['date','time','close','volume']),ignore_index=True)

print(csvpd.head())
csvpd.to_csv('output.csv',index=False)
gc.set_threshold(700, 10, 5)
end=time.time()
elapsed = end - start
print('運行時間: ',elapsed)
Example #38
0
import mod_openopc_common
__version__ = mod_openopc_common.VERSION
OPC_STATUS = mod_openopc_common.OPC_STATUS
BROWSER_TYPE = mod_openopc_common.BROWSER_TYPE
ACCESS_RIGHTS = mod_openopc_common.ACCESS_RIGHTS
OPC_QUALITY = mod_openopc_common.OPC_QUALITY
OPC_CLASS = mod_openopc_common.OPC_CLASS
OPC_SERVER = mod_openopc_common.OPC_SERVER
OPC_CLIENT = mod_openopc_common.OPC_CLIENT
OPC_GATE_PORT = mod_openopc_common.OPC_GATEWAY_PORT
mod_openopc_GARBAGE_0 = mod_openopc_common.GARBAGE_0
mod_openopc_GARBAGE_1 = mod_openopc_common.GARBAGE_1
mod_openopc_GARBAGE_2 = mod_openopc_common.GARBAGE_2
# --------------------- -- GARBAGE COLLECTION -----------------------
import gc
gc.set_threshold(mod_openopc_GARBAGE_0, mod_openopc_GARBAGE_1,
                 mod_openopc_GARBAGE_2)
gc.enable()
# --------------------- LOAD LIBRARIES ------------------------------
# --------------------- -- STANDARD ---------------------------------
import os
import sys
import time
import types
import string
import socket
import re
import Queue
# --------------------- -- DEPENDANCIES -----------------------------
import Pyro
import Pyro.core
import Pyro.protocol
Example #39
0
        # end process, close CrossEPG DB saving data
        crossdb.close_db()
        self.log("TOTAL EPG EVENTS PROCESSED: %d" % total_events)
        self.log("--- END ---")
        self.log2video("END , events processed: %d" % total_events)


# ****************************************************************************************************************************

# MAIN CODE: SCRIPT START HERE
# increase this process niceness (other processes have higher priority)
os.nice(10)

# set Garbage Collector to do a "generational jump" more frequently than default 700
# memory saving: about 50% (!!), some performance loss (obviously)
gc.set_threshold(50, 10, 10)

SCRIPT_DIR = 'scripts/mediaprem/'

# get CrossEPG installation dir.
crossepg_instroot = crossepg.epgdb_get_installroot()
if crossepg_instroot == False:
    sys.exit(1)
scriptlocation = os.path.join(crossepg_instroot, SCRIPT_DIR)

# get where CrossEPG save data (dbroot) and use it as script cache repository
crossepg_dbroot = crossepg.epgdb_get_dbroot()
if crossepg_dbroot == False:
    sys.exit(1)

# initialize script class
Example #40
0
 def enable_gc():
     """enable garbage collection"""
     gc.set_threshold(700)
Example #41
0
def main():
    opt = parse_args()

    makedirs(opt.save_dir)

    filehandler = logging.FileHandler(
        os.path.join(opt.save_dir, opt.logging_file))
    streamhandler = logging.StreamHandler()
    logger = logging.getLogger('')
    logger.setLevel(logging.INFO)
    logger.addHandler(filehandler)
    logger.addHandler(streamhandler)
    logger.info(opt)

    gc.set_threshold(100, 5, 5)

    # set env
    if opt.gpu_id == -1:
        context = mx.cpu()
    else:
        gpu_id = opt.gpu_id
        context = mx.gpu(gpu_id)

    # get data preprocess
    image_norm_mean = [0.485, 0.456, 0.406]
    image_norm_std = [0.229, 0.224, 0.225]
    if opt.ten_crop:
        transform_test = transforms.Compose([
            video.VideoTenCrop(opt.input_size),
            video.VideoToTensor(),
            video.VideoNormalize(image_norm_mean, image_norm_std)
        ])
        opt.num_crop = 10
    elif opt.three_crop:
        transform_test = transforms.Compose([
            video.VideoThreeCrop(opt.input_size),
            video.VideoToTensor(),
            video.VideoNormalize(image_norm_mean, image_norm_std)
        ])
        opt.num_crop = 3
    else:
        transform_test = video.VideoGroupValTransform(size=opt.input_size,
                                                      mean=image_norm_mean,
                                                      std=image_norm_std)
        opt.num_crop = 1

    # get model
    if opt.use_pretrained and len(opt.hashtag) > 0:
        opt.use_pretrained = opt.hashtag
    classes = opt.num_classes
    model_name = opt.model
    net = get_model(name=model_name,
                    nclass=classes,
                    pretrained=opt.use_pretrained,
                    num_segments=opt.num_segments,
                    num_crop=opt.num_crop)
    net.cast(opt.dtype)
    net.collect_params().reset_ctx(context)
    if opt.mode == 'hybrid':
        net.hybridize(static_alloc=True, static_shape=True)
    if opt.resume_params != '' and not opt.use_pretrained:
        net.load_parameters(opt.resume_params, ctx=context)
        logger.info('Pre-trained model %s is successfully loaded.' %
                    (opt.resume_params))
    else:
        logger.info(
            'Pre-trained model is successfully loaded from the model zoo.')
    logger.info("Successfully built model {}".format(model_name))

    # get classes list, if we are using a pretrained network from the model_zoo
    classes = None
    if opt.use_pretrained:
        if "kinetics400" in model_name:
            classes = Kinetics400Attr().classes
        elif "ucf101" in model_name:
            classes = UCF101Attr().classes
        elif "hmdb51" in model_name:
            classes = HMDB51Attr().classes
        elif "sthsth" in model_name:
            classes = SomethingSomethingV2Attr().classes

    # get data
    anno_file = opt.data_list
    f = open(anno_file, 'r')
    data_list = f.readlines()
    logger.info('Load %d video samples.' % len(data_list))

    # build a pseudo dataset instance to use its children class methods
    video_utils = VideoClsCustom(root=opt.data_dir,
                                 setting=opt.data_list,
                                 num_segments=opt.num_segments,
                                 num_crop=opt.num_crop,
                                 new_length=opt.new_length,
                                 new_step=opt.new_step,
                                 new_width=opt.new_width,
                                 new_height=opt.new_height,
                                 video_loader=opt.video_loader,
                                 use_decord=opt.use_decord,
                                 slowfast=opt.slowfast,
                                 slow_temporal_stride=opt.slow_temporal_stride,
                                 fast_temporal_stride=opt.fast_temporal_stride,
                                 data_aug=opt.data_aug,
                                 lazy_init=True)

    start_time = time.time()
    for vid, vline in enumerate(data_list):
        video_path = vline.split()[0]
        video_name = video_path.split('/')[-1]
        if opt.need_root:
            video_path = os.path.join(opt.data_dir, video_path)
        video_data = read_data(opt, video_path, transform_test, video_utils)
        video_input = video_data.as_in_context(context)
        pred = net(video_input.astype(opt.dtype, copy=False))
        if opt.save_logits:
            logits_file = '%s_%s_logits.npy' % (model_name, video_name)
            np.save(os.path.join(opt.save_dir, logits_file), pred.asnumpy())
        pred_label = np.argmax(pred.asnumpy())
        if opt.save_preds:
            preds_file = '%s_%s_preds.npy' % (model_name, video_name)
            np.save(os.path.join(opt.save_dir, preds_file), pred_label)

        # Try to report a text label instead of the number.
        if classes:
            pred_label = classes[pred_label]

        logger.info('%04d/%04d: %s is predicted to class %s' %
                    (vid, len(data_list), video_name, pred_label))

    end_time = time.time()
    logger.info('Total inference time is %4.2f minutes' %
                ((end_time - start_time) / 60))
Example #42
0
def setup_tests(ns):
    try:
        stderr_fd = sys.__stderr__.fileno()
    except (ValueError, AttributeError):
        # Catch ValueError to catch io.UnsupportedOperation on TextIOBase
        # and ValueError on a closed stream.
        #
        # Catch AttributeError for stderr being None.
        stderr_fd = None
    else:
        # Display the Python traceback on fatal errors (e.g. segfault)
        faulthandler.enable(all_threads=True, file=stderr_fd)

        # Display the Python traceback on SIGALRM or SIGUSR1 signal
        signals = []
        if hasattr(signal, 'SIGALRM'):
            signals.append(signal.SIGALRM)
        if hasattr(signal, 'SIGUSR1'):
            signals.append(signal.SIGUSR1)
        for signum in signals:
            faulthandler.register(signum, chain=True, file=stderr_fd)

    replace_stdout()
    support.record_original_stdout(sys.stdout)

    if ns.testdir:
        # Prepend test directory to sys.path, so runtest() will be able
        # to locate tests
        sys.path.insert(0, os.path.abspath(ns.testdir))

    # Some times __path__ and __file__ are not absolute (e.g. while running from
    # Lib/) and, if we change the CWD to run the tests in a temporary dir, some
    # imports might fail.  This affects only the modules imported before os.chdir().
    # These modules are searched first in sys.path[0] (so '' -- the CWD) and if
    # they are found in the CWD their __file__ and __path__ will be relative (this
    # happens before the chdir).  All the modules imported after the chdir, are
    # not found in the CWD, and since the other paths in sys.path[1:] are absolute
    # (site.py absolutize them), the __file__ and __path__ will be absolute too.
    # Therefore it is necessary to absolutize manually the __file__ and __path__ of
    # the packages to prevent later imports to fail when the CWD is different.
    for module in sys.modules.values():
        if hasattr(module, '__path__'):
            for index, path in enumerate(module.__path__):
                module.__path__[index] = os.path.abspath(path)
        if getattr(module, '__file__', None):
            module.__file__ = os.path.abspath(module.__file__)

    if ns.huntrleaks:
        unittest.BaseTestSuite._cleanup = False

        # Avoid false positives due to various caches
        # filling slowly with random data:
        warm_caches()

    if ns.memlimit is not None:
        support.set_memlimit(ns.memlimit)

    if ns.threshold is not None:
        gc.set_threshold(ns.threshold)

    try:
        import msvcrt
    except ImportError:
        pass
    else:
        msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS|
                            msvcrt.SEM_NOALIGNMENTFAULTEXCEPT|
                            msvcrt.SEM_NOGPFAULTERRORBOX|
                            msvcrt.SEM_NOOPENFILEERRORBOX)
        try:
            msvcrt.CrtSetReportMode
        except AttributeError:
            # release build
            pass
        else:
            for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]:
                if ns.verbose and ns.verbose >= 2:
                    msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE)
                    msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR)
                else:
                    msvcrt.CrtSetReportMode(m, 0)

    support.use_resources = ns.use_resources
Example #43
0
def main(driver):
    gc.enable()
    gc.set_threshold(50, 10, 10)
    WebDriverWait(driver, 5).until(
        EC.element_to_be_clickable(
            (By.ID, "secondaryCurrencyProgressWrapper"))).click()
    time.sleep(1.5)
    while True:
        if len(
                WebDriverWait(driver, 5).until(
                    EC.presence_of_element_located(
                        (By.ID, "secondaryCurrencyScratchOffWrapper"
                         ))).get_attribute("style")) is 14:
            WebDriverWait(driver, 5).until(
                EC.element_to_be_clickable(
                    (By.ID, "secondaryCurrencyProgressWrapper"))).click()
        center_game(driver)
        if pyautogui.locateCenterOnScreen("top left.png",
                                          grayscale=True) != None:
            x1, y1 = pyautogui.locateCenterOnScreen("top left.png",
                                                    grayscale=True)
            if pyautogui.pixelMatchesColor(int(x1 + 61),
                                           int(y1 + 63), (43, 234, 246),
                                           tolerance=25):
                pyautogui.click(x1 + 234, y1 + 303, duration=0.15)
            break
        else:
            if pyautogui.locateOnScreen("play button.png") != None:
                pyautogui.click(
                    pyautogui.locateCenterOnScreen("play button.png"),
                    duration=0.15)
    print((x1, y1))
    side_length = 385
    counter = 0
    counter1 = 0
    counter2 = 0
    timer = time.time()
    timer_stop = timer + random.randint(3600, 4200)
    past_time = time.time()
    while True:
        if time.time() > timer_stop:
            print("Stopped at " + str((timer_stop / 60) / 60) + " hours")
            return
        center_game(driver)
        try:
            WebDriverWait(driver, 0).until(
                EC.visibility_of_element_located((By.ID, "error-window")))
            go_candy_jam(driver)
            print("reset")
        except:
            pass
        rows_color, rows_pos = horizontials(x1 + 27, y1 + 29)
        #Use selenium for scratch and win
        if len(
                WebDriverWait(driver, 0).until(
                    EC.presence_of_element_located(
                        (By.ID, "scratchOffWrapperTier-1"
                         ))).get_attribute("class")) is 8 and counter1 is 0:
            print(
                str(int((time.time() - past_time) / 60)) + " min" +
                " - Tier 1")
            center_game(driver)
            counter1 += 1
        if len(
                WebDriverWait(driver, 0).until(
                    EC.presence_of_element_located(
                        (By.ID, "scratchOffWrapperTier-2"
                         ))).get_attribute("class")) is 8 and counter2 is 0:
            print(
                str(int((time.time() - past_time) / 60)) + " min" +
                " - Tier 2")
            center_game(driver)
            #counter2 += 1
            WebDriverWait(driver, 5).until(
                EC.element_to_be_clickable(
                    (By.ID, "scratchNowButtonTier-2"))).click()
            scratch_all_tier_2(driver)
            WebDriverWait(driver, 30).until(
                EC.element_to_be_clickable(
                    (By.ID, "scYouWinEarnMoreButton"))).click()
            counter1 = 0
            past_time = time.time()
        if len(
                WebDriverWait(driver, 0).until(
                    EC.presence_of_element_located(
                        (By.ID, "secondaryCurrencyScratchOffWrapper"
                         ))).get_attribute("style")) is 14:
            WebDriverWait(driver, 5).until(
                EC.element_to_be_clickable(
                    (By.ID, "secondaryCurrencyProgressWrapper"))).click()
        if pyautogui.pixelMatchesColor(int(x1 + 61),
                                       int(y1 + 63), (43, 234, 246),
                                       tolerance=25):
            pyautogui.click(x1 + 234, y1 + 303, duration=0.15)
        if pyautogui.pixelMatchesColor(int(x1 + 195),
                                       int(y1 + 189), (255, 151, 7),
                                       tolerance=10):
            pyautogui.click(int(x1 + 195), int(y1 + 189), duration=0.15)
        #print("next_rows")
        if do_next_rows(rows_color, rows_pos) is False:
            #print("next verticals")
            if do_next_verticals(rows_color, rows_pos) is False:
                #print("skipped rows")
                if do_skipped_rows(rows_color, rows_pos) is False:
                    #print("skipped verticals")
                    if do_skipped_verticals(rows_color, rows_pos) is False:
                        if bomb(rows_color, rows_pos) is False:
                            if counter >= 10:
                                print("no solutions")
                                pyautogui.screenshot("ending picture.png")
                                counter = 0
                            else:
                                counter += 1
        else:
            counter = 0
Example #44
0
def main(logger):
    opt = parse_args()

    makedirs(opt.save_dir)

    filehandler = logging.FileHandler(
        os.path.join(opt.save_dir, opt.logging_file))
    streamhandler = logging.StreamHandler()
    logger = logging.getLogger('')
    logger.setLevel(logging.INFO)
    logger.addHandler(filehandler)
    logger.addHandler(streamhandler)
    logger.info(opt)

    gc.set_threshold(100, 5, 5)

    # set env
    gpu_id = opt.gpu_id
    context = mx.gpu(gpu_id)

    # get data preprocess
    image_norm_mean = [0.485, 0.456, 0.406]
    image_norm_std = [0.229, 0.224, 0.225]
    if opt.ten_crop:
        transform_test = transforms.Compose([
            video.VideoTenCrop(opt.input_size),
            video.VideoToTensor(),
            video.VideoNormalize(image_norm_mean, image_norm_std)
        ])
        opt.num_crop = 10
    elif opt.three_crop:
        transform_test = transforms.Compose([
            video.VideoThreeCrop(opt.input_size),
            video.VideoToTensor(),
            video.VideoNormalize(image_norm_mean, image_norm_std)
        ])
        opt.num_crop = 3
    else:
        transform_test = video.VideoGroupValTransform(size=opt.input_size,
                                                      mean=image_norm_mean,
                                                      std=image_norm_std)
        opt.num_crop = 1

    # get model
    if opt.use_pretrained and len(opt.hashtag) > 0:
        opt.use_pretrained = opt.hashtag
    classes = opt.num_classes
    model_name = opt.model
    net = get_model(name=model_name,
                    nclass=classes,
                    pretrained=opt.use_pretrained,
                    num_segments=opt.num_segments,
                    num_crop=opt.num_crop)
    net.cast(opt.dtype)
    net.collect_params().reset_ctx(context)
    if opt.mode == 'hybrid':
        net.hybridize(static_alloc=True, static_shape=True)
    if opt.resume_params is not '' and not opt.use_pretrained:
        net.load_parameters(opt.resume_params, ctx=context)
        logger.info('Pre-trained model %s is successfully loaded.' %
                    (opt.resume_params))
    else:
        logger.info(
            'Pre-trained model is successfully loaded from the model zoo.')
    logger.info("Successfully built model {}".format(model_name))

    # get data
    anno_file = opt.data_list
    f = open(anno_file, 'r')
    data_list = f.readlines()
    logger.info('Load %d video samples.' % len(data_list))

    start_time = time.time()
    for vid, vline in enumerate(data_list):
        video_path = vline.split()[0]
        video_name = video_path.split('/')[-1]
        if opt.need_root:
            video_path = os.path.join(opt.data_dir, video_path)
        video_data = read_data(opt, video_path, transform_test)
        video_input = video_data.as_in_context(context)
        pred = net(video_input.astype(opt.dtype, copy=False))
        if opt.save_logits:
            logits_file = '%s_%s_logits.npy' % (model_name, video_name)
            np.save(os.path.join(opt.save_dir, logits_file), pred.asnumpy())
        pred_label = np.argmax(pred.asnumpy())
        if opt.save_preds:
            preds_file = '%s_%s_preds.npy' % (model_name, video_name)
            np.save(os.path.join(opt.save_dir, preds_file), pred_label)

        logger.info('%04d/%04d: %s is predicted to class %d' %
                    (vid, len(data_list), video_name, pred_label))

    end_time = time.time()
    logger.info('Total inference time is %4.2f minutes' %
                ((end_time - start_time) / 60))
Example #45
0
def main(scheduler, host, worker_port, listen_address, contact_address,
         nanny_port, nthreads, nprocs, nanny, name, pid_file, resources,
         dashboard, bokeh, bokeh_port, scheduler_file, dashboard_prefix,
         tls_ca_file, tls_cert, tls_key, dashboard_address, worker_class,
         preload_nanny, **kwargs):
    g0, g1, g2 = gc.get_threshold(
    )  # https://github.com/dask/distributed/issues/1653
    gc.set_threshold(g0 * 3, g1 * 3, g2 * 3)

    enable_proctitle_on_current()
    enable_proctitle_on_children()

    if bokeh_port is not None:
        warnings.warn(
            "The --bokeh-port flag has been renamed to --dashboard-address. "
            "Consider adding ``--dashboard-address :%d`` " % bokeh_port)
        dashboard_address = bokeh_port
    if bokeh is not None:
        warnings.warn(
            "The --bokeh/--no-bokeh flag has been renamed to --dashboard/--no-dashboard. "
        )
        dashboard = bokeh

    sec = {
        k: v
        for k, v in [
            ("tls_ca_file", tls_ca_file),
            ("tls_worker_cert", tls_cert),
            ("tls_worker_key", tls_key),
        ] if v is not None
    }

    if nprocs == "auto":
        nprocs, nthreads = nprocesses_nthreads()
    else:
        nprocs = int(nprocs)

    if nprocs < 0:
        nprocs = CPU_COUNT + 1 + nprocs

    if nprocs <= 0:
        logger.error(
            "Failed to launch worker. Must specify --nprocs so that there's at least one process."
        )
        sys.exit(1)

    if nprocs > 1 and not nanny:
        logger.error(
            "Failed to launch worker.  You cannot use the --no-nanny argument when nprocs > 1."
        )
        sys.exit(1)

    if contact_address and not listen_address:
        logger.error(
            "Failed to launch worker. "
            "Must specify --listen-address when --contact-address is given")
        sys.exit(1)

    if nprocs > 1 and listen_address:
        logger.error("Failed to launch worker. "
                     "You cannot specify --listen-address when nprocs > 1.")
        sys.exit(1)

    if (worker_port or host) and listen_address:
        logger.error(
            "Failed to launch worker. "
            "You cannot specify --listen-address when --worker-port or --host is given."
        )
        sys.exit(1)

    try:
        if listen_address:
            (host, worker_port) = get_address_host_port(listen_address,
                                                        strict=True)

        if contact_address:
            # we only need this to verify it is getting parsed
            (_, _) = get_address_host_port(contact_address, strict=True)
        else:
            # if contact address is not present we use the listen_address for contact
            contact_address = listen_address
    except ValueError as e:
        logger.error("Failed to launch worker. " + str(e))
        sys.exit(1)

    if nanny:
        port = nanny_port
    else:
        port = worker_port

    if not nthreads:
        nthreads = CPU_COUNT // nprocs

    if pid_file:
        with open(pid_file, "w") as f:
            f.write(str(os.getpid()))

        def del_pid_file():
            if os.path.exists(pid_file):
                os.remove(pid_file)

        atexit.register(del_pid_file)

    if resources:
        resources = resources.replace(",", " ").split()
        resources = dict(pair.split("=") for pair in resources)
        resources = valmap(float, resources)
    else:
        resources = None

    loop = IOLoop.current()

    worker_class = import_term(worker_class)
    if nanny:
        kwargs["worker_class"] = worker_class
        kwargs["preload_nanny"] = preload_nanny

    if nanny:
        kwargs.update({
            "worker_port": worker_port,
            "listen_address": listen_address
        })
        t = Nanny
    else:
        if nanny_port:
            kwargs["service_ports"] = {"nanny": nanny_port}
        t = worker_class

    if (not scheduler and not scheduler_file
            and dask.config.get("scheduler-address", None) is None):
        raise ValueError("Need to provide scheduler address like\n"
                         "dask-worker SCHEDULER_ADDRESS:8786")

    with suppress(TypeError, ValueError):
        name = int(name)

    nannies = [
        t(scheduler,
          scheduler_file=scheduler_file,
          nthreads=nthreads,
          loop=loop,
          resources=resources,
          security=sec,
          contact_address=contact_address,
          host=host,
          port=port,
          dashboard=dashboard,
          dashboard_address=dashboard_address,
          name=name if nprocs == 1 or name is None or name == "" else
          str(name) + "-" + str(i),
          **kwargs) for i in range(nprocs)
    ]

    async def close_all():
        # Unregister all workers from scheduler
        if nanny:
            await asyncio.gather(*[n.close(timeout=2) for n in nannies])

    signal_fired = False

    def on_signal(signum):
        nonlocal signal_fired
        signal_fired = True
        if signum != signal.SIGINT:
            logger.info("Exiting on signal %d", signum)
        return asyncio.ensure_future(close_all())

    async def run():
        await asyncio.gather(*nannies)
        await asyncio.gather(*[n.finished() for n in nannies])

    install_signal_handlers(loop, cleanup=on_signal)

    try:
        loop.run_sync(run)
    except TimeoutError:
        # We already log the exception in nanny / worker. Don't do it again.
        if not signal_fired:
            logger.info("Timed out starting worker")
        sys.exit(1)
    except KeyboardInterrupt:
        pass
    finally:
        logger.info("End worker")
Example #46
0
import fsps
from powderday.image_processing import add_transmission_filters, convolve
from powderday.m_control_tools import m_control_sph, m_control_enzo, m_control_arepo
import powderday.backwards_compatibility as bc
import powderday.error_handling as eh
import powderday.SED_gen as sg
from powderday.front_ends.front_end_controller import stream
from astropy import units as u
import powderday.config as cfg
import h5py
import matplotlib as mpl
import copy
import numpy as np
import sys
import gc
gc.set_threshold(0)

script, pardir, parfile, modelfile = sys.argv

mpl.use('Agg')

sys.path.insert(0, pardir)
par = __import__(parfile)
model = __import__(modelfile)

cfg.par = par  # re-write cfg.par for all modules that read this in now
cfg.model = model

# =========================================================
# CHECK FOR THE EXISTENCE OF A FEW CRUCIAL FILES FIRST
# =========================================================
Example #47
0
# ====================

# Import required modules
from collections import OrderedDict
import gc
import getopt
import locale
import sys
from amed.amed_pre import *
from amed.amed_post import *

# Set locale to assist with sorting
locale.setlocale(locale.LC_ALL, '')

# Set threshold for garbage collection (helps prevent the program run out of memory)
gc.set_threshold(400, 5, 5)

__author__ = 'Victoria Morris'
__license__ = 'MIT License'
__version__ = '1.0.0'
__status__ = '4 - Beta Development'

# ====================
#     Constants
# ====================

OPTIONS = OrderedDict([
    ('E1', ('Prepare AMED files for import to Excel', amed_pre_Excel)),
    ('E2', ('Process AMED files exported from Excel', amed_post_Excel)),
    ('LM1', ('Prepare AMED files for import to Library Master', amed_pre_LM)),
    ('LM2', ('Process AMED files exported from Library Master', amed_post_LM)),
Example #48
0
 def startService(self):
     hs = setup(self.config)
     change_resource_limit(hs.config.soft_file_limit)
     if hs.config.gc_thresholds:
         gc.set_threshold(*hs.config.gc_thresholds)
def setup_tests(ns):
    # Display the Python traceback on fatal errors (e.g. segfault)
    faulthandler.enable(all_threads=True)

    # Display the Python traceback on SIGALRM or SIGUSR1 signal
    signals = []
    if hasattr(signal, 'SIGALRM'):
        signals.append(signal.SIGALRM)
    if hasattr(signal, 'SIGUSR1'):
        signals.append(signal.SIGUSR1)
    for signum in signals:
        faulthandler.register(signum, chain=True)

    replace_stdout()
    support.record_original_stdout(sys.stdout)

    if ns.testdir:
        # Prepend test directory to sys.path, so runtest() will be able
        # to locate tests
        sys.path.insert(0, os.path.abspath(ns.testdir))

    # Some times __path__ and __file__ are not absolute (e.g. while running from
    # Lib/) and, if we change the CWD to run the tests in a temporary dir, some
    # imports might fail.  This affects only the modules imported before os.chdir().
    # These modules are searched first in sys.path[0] (so '' -- the CWD) and if
    # they are found in the CWD their __file__ and __path__ will be relative (this
    # happens before the chdir).  All the modules imported after the chdir, are
    # not found in the CWD, and since the other paths in sys.path[1:] are absolute
    # (site.py absolutize them), the __file__ and __path__ will be absolute too.
    # Therefore it is necessary to absolutize manually the __file__ and __path__ of
    # the packages to prevent later imports to fail when the CWD is different.
    for module in sys.modules.values():
        if hasattr(module, '__path__'):
            for index, path in enumerate(module.__path__):
                module.__path__[index] = os.path.abspath(path)
        if hasattr(module, '__file__'):
            module.__file__ = os.path.abspath(module.__file__)

    # MacOSX (a.k.a. Darwin) has a default stack size that is too small
    # for deeply recursive regular expressions.  We see this as crashes in
    # the Python test suite when running test_re.py and test_sre.py.  The
    # fix is to set the stack limit to 2048.
    # This approach may also be useful for other Unixy platforms that
    # suffer from small default stack limits.
    if sys.platform == 'darwin':
        try:
            import resource
        except ImportError:
            pass
        else:
            soft, hard = resource.getrlimit(resource.RLIMIT_STACK)
            newsoft = min(hard, max(soft, 1024 * 2048))
            resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard))

    if ns.huntrleaks:
        unittest.BaseTestSuite._cleanup = False

        # Avoid false positives due to various caches
        # filling slowly with random data:
        warm_caches()

    if ns.memlimit is not None:
        support.set_memlimit(ns.memlimit)

    if ns.threshold is not None:
        gc.set_threshold(ns.threshold)

    try:
        import msvcrt
    except ImportError:
        pass
    else:
        msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS
                            | msvcrt.SEM_NOALIGNMENTFAULTEXCEPT
                            | msvcrt.SEM_NOGPFAULTERRORBOX
                            | msvcrt.SEM_NOOPENFILEERRORBOX)
        try:
            msvcrt.CrtSetReportMode
        except AttributeError:
            # release build
            pass
        else:
            for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]:
                if ns.verbose and ns.verbose >= 2:
                    msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE)
                    msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR)
                else:
                    msvcrt.CrtSetReportMode(m, 0)

    support.use_resources = ns.use_resources
Example #50
0
try:
    threshold = int(sys.argv[1])
except (IndexError, ValueError, TypeError):
    print('Missing or invalid threshold, using default')
    threshold = 5


class MyObj:
    def __init__(self, name):
        self.name = name
        print('Created', self.name)


gc.set_debug(gc.DEBUG_STATS)

gc.set_threshold(threshold, 1, 1)
print('Thresholds:', gc.get_threshold())

print('Clear the collector by forcing a run')
gc.collect()
print()

print('Creating objects')
objs = []
for i in range(10):
    objs.append(MyObj(i))
print('Exiting')

# Turn off debugging
gc.set_debug(0)
Example #51
0
def gc_threshold_demo():
    print 'gc threshold:', gc.get_threshold()
    gc.set_threshold(500, 10, 5)
    print "gc threshold (after 'gc.set_threshold(500, 10, 5)'):", gc.get_threshold()
Example #52
0
def setup_tests(ns):
    try:
        stderr_fd = sys.__stderr__.fileno()
    except (ValueError, AttributeError):
        # Catch ValueError to catch io.UnsupportedOperation on TextIOBase
        # and ValueError on a closed stream.
        #
        # Catch AttributeError for stderr being None.
        stderr_fd = None
    else:
        # Display the Python traceback on fatal errors (e.g. segfault)
        faulthandler.enable(all_threads=True, file=stderr_fd)

        # Display the Python traceback on SIGALRM or SIGUSR1 signal
        signals = []
        if hasattr(signal, 'SIGALRM'):
            signals.append(signal.SIGALRM)
        if hasattr(signal, 'SIGUSR1'):
            signals.append(signal.SIGUSR1)
        for signum in signals:
            faulthandler.register(signum, chain=True, file=stderr_fd)

    replace_stdout()
    support.record_original_stdout(sys.stdout)

    if ns.testdir:
        # Prepend test directory to sys.path, so runtest() will be able
        # to locate tests
        sys.path.insert(0, os.path.abspath(ns.testdir))

    # Some times __path__ and __file__ are not absolute (e.g. while running from
    # Lib/) and, if we change the CWD to run the tests in a temporary dir, some
    # imports might fail.  This affects only the modules imported before os.chdir().
    # These modules are searched first in sys.path[0] (so '' -- the CWD) and if
    # they are found in the CWD their __file__ and __path__ will be relative (this
    # happens before the chdir).  All the modules imported after the chdir, are
    # not found in the CWD, and since the other paths in sys.path[1:] are absolute
    # (site.py absolutize them), the __file__ and __path__ will be absolute too.
    # Therefore it is necessary to absolutize manually the __file__ and __path__ of
    # the packages to prevent later imports to fail when the CWD is different.
    for module in sys.modules.values():
        if hasattr(module, '__path__'):
            for index, path in enumerate(module.__path__):
                module.__path__[index] = os.path.abspath(path)
        if getattr(module, '__file__', None):
            module.__file__ = os.path.abspath(module.__file__)

    # MacOSX (a.k.a. Darwin) has a default stack size that is too small
    # for deeply recursive regular expressions.  We see this as crashes in
    # the Python test suite when running test_re.py and test_sre.py.  The
    # fix is to set the stack limit to 2048.
    # This approach may also be useful for other Unixy platforms that
    # suffer from small default stack limits.
    if sys.platform == 'darwin':
        try:
            import resource
        except ImportError:
            pass
        else:
            soft, hard = resource.getrlimit(resource.RLIMIT_STACK)
            newsoft = min(hard, max(soft, 1024 * 2048))
            resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard))

    if ns.huntrleaks:
        unittest.BaseTestSuite._cleanup = False

    if ns.memlimit is not None:
        support.set_memlimit(ns.memlimit)

    if ns.threshold is not None:
        gc.set_threshold(ns.threshold)

    suppress_msvcrt_asserts(ns.verbose and ns.verbose >= 2)

    support.use_resources = ns.use_resources

    if hasattr(sys, 'addaudithook'):
        # Add an auditing hook for all tests to ensure PySys_Audit is tested
        def _test_audit_hook(name, args):
            pass

        sys.addaudithook(_test_audit_hook)
Example #53
0
import gc, inspect, types, traceback, threading, time, pprint
from pyGlobus.io import GSITCPSocket
from AccessGrid.Security.Utilities import CreateTCPAttrAlwaysAuth

gc.set_debug(gc.DEBUG_LEAK)
print "GC Threshold: ", gc.get_threshold()
gc.set_threshold(1, 1, 1)


def Info(obj):
    if inspect.ismethod(obj):
        print "Method (%d): " % id(obj)
        print " Class: ", obj.im_class
        print " Name: ", obj.__name__
    elif isinstance(obj, types.InstanceType):
        print "Instance (%d): " % id(obj)
        pprint.pprint(obj)
    elif inspect.isframe(obj):
        print "Frame (%d): " % id(obj)
        traceback.print_stack(obj)
    else:
        print "Other (%d): " % id(obj)
        print type(obj)


#        pprint.pprint(obj)
    print " ---- "


class EventService:
    """
Example #54
0
import logging.handlers
import gc
import re

#agent base directory
exe_path = os.path.dirname(os.path.realpath(__file__)).rsplit('/', 1)[0]

import helper
import controller
import api
import exit_status
from universal import Universal
from constructs import *

_log = logging.getLogger(__name__)  #module level logging
gc.set_threshold(50, 5, 5)  #set gc threshold
logging_level = logging.INFO  #default logging level

#setup logging for StreamHandler
#in StreamHandler we display only messages till everything initializes. This is done not polute the terminal when running as service
logging.basicConfig(level=logging_level, format='%(message)s')

formatter = logging.Formatter(
    '%(asctime)-15s %(levelname)-7s %(thread)d - %(module)-s[%(lineno)-d]: %(message)s'
)  #formatter instance for logging
logger = logging.getLogger()  #get the root logger

try:
    #create rotating file logger and add to root logger
    #this can raise exception if the file cannot be created
    lf = logging.handlers.RotatingFileHandler(
Example #55
0
        hope_mvector = theoracle.clean(hope_vector)

        if log.level >= 1:
            log.write("hope hyp: %s\n" %
                      " ".join(sym.tostring(e) for e in hope))
            log.write("hope features: %s\n" % hope_mvector)
            log.write("hope oracle: %s\n" % hope_ovector)
        return maxmargin.Hypothesis(hope_mvector, hope_ovector)


def make_oracle():
    return oracle.Oracle(order=4, variant=opts.bleuvariant, oracledoc_size=10)


if __name__ == "__main__":
    gc.set_threshold(100000, 10, 10)

    import optparse
    optparser = optparse.OptionParser()
    optparser.add_option("-W",
                         dest="outweightfilename",
                         help="filename to write weights to")
    optparser.add_option("-L",
                         dest="outscorefilename",
                         help="filename to write BLEU scores to")
    optparser.add_option("-B", dest="bleuvariant", default="NIST")
    optparser.add_option("-S", dest="stopfile")
    optparser.add_option("--input-lattice",
                         dest="input_lattice",
                         action="store_true")
    optparser.add_option("--holdout",
def testThreshold(t0=700, t1=10, t2=10):
    gc.set_threshold(t0, t1, t2)
    for i in range(1000000):
        a = A()
    gc.collect()
Example #57
0
def main():
	parser = argparse.ArgumentParser(
		description="access to pythons built-in garbage collector")
	parser.add_argument(
		"command", help="what to do",
		choices=[
			"enable", "disable", "status", "collect",
			"threshold", "debug", "break",
			],
		action="store")
	parser.add_argument(
		"args", help="argument for command", action="store", nargs="*"
		)
	ns = parser.parse_args()
	if ns.command == "enable":
		gc.enable()
	elif ns.command == "disable":
		gc.disable()
	elif ns.command == "collect":
		gc.collect()
	elif ns.command == "status":
		if gc.isenabled():
			gcs = _stash.text_color("Enabled", "green")
		else:
			gcs = _stash.text_color("Disabled", "red")
		sys.stdout.write("GC status:               {s}\n".format(s=gcs))
		tracked = gc.get_objects()
		n = len(tracked)
		sys.stdout.write("Tracked objects:         {n}\n".format(n=n))
		size = sum([sys.getsizeof(e) for e in tracked])
		del tracked  # this list may be big, better delete it
		sys.stdout.write("Size of tracked objects: {s} bytes\n".format(s=size))
		sys.stdout.write("Garbage:                 {n}\n".format(n=len(gc.garbage)))
		gsize = sum([sys.getsizeof(e) for e in gc.garbage])
		sys.stdout.write("Size of garbage:         {s} bytes\n".format(s=gsize))
		sys.stdout.write("Debug:                   {d}\n".format(d=gc.get_debug()))
	elif ns.command == "threshold":
		if len(ns.args) == 0:
			sys.stdout.write(
				"Threshold:\n   G1: {}\n   G2: {}\n   G3: {}\n".format(*gc.get_threshold())
				)
		elif len(ns.args) > 3:
			errmsg = _stash.text_color(
				"Error: to many arguments for threshold!\n", "red"
				)
			sys.stdout.write(errmsg)
			sys.exit(1)
		else:
			try:
				ts = tuple([int(e) for e in ns.args])
			except ValueError:
				errmsg = _stash.text_color(
					"Error: expected arguments to be integer!\n", "red"
					)
				sys.stdout.write(errmsg)
				sys.exit(1)
			gc.set_threshold(*ts)
	elif ns.command == "debug":
		if len(ns.args) == 0:
			sys.stdout.write("Debug: {d}\n".format(d=gc.get_debug()))
		elif len(ns.args) == 1:
			try:
				flag = int(ns.args[0])
			except ValueError:
				sys.stdout.write(
					_stash.text_color("Error: expected argument to be an integer!\n", "red")
					)
				sys.exit(1)
			gc.set_debug(flag)
		else:
			sys.stdout.write(
				_stash.text_color(
					"Error: expected exactly one argument for debug!\n", "red"
					)
				)
			sys.exit(1)
	elif ns.command == "break":
		if len(gc.garbage) == 0:
			sys.stdout.write(
				_stash.text_color("Error: No Garbage found!\n", "red")
				)
			sys.exit(1)
		else:
			for k in dir(gc.garbage[0]):
				try:
					delattr(gc.garbage[0], k)
				except:
					pass
			del gc.garbage[:]
Example #58
0
# 2. 按归属放入报警队列alert_queue
#

import sys
import gc
import time, shutil, os
import helper
from config import setting

db = setting.db_web

if __name__ == '__main__':

    print "DETECTOR: %s started" % helper.time_str()

    gc.set_threshold(300, 5, 5)

    try:
        _count = _dir = _ins = 0
        while 1:
            # 检查ftp目录
            _count += 1
            ftp_dir = os.listdir(setting.ftp_path)
            for ip in ftp_dir:
                _dir += 1
                db_cam = db.cams.find_one({'cam_ip': ip}, {
                    'motion_detect': 1,
                    'owner': 1
                })

                if db_cam != None and db_cam['motion_detect'] > 0:
Example #59
0
# -*- coding: utf-8 -*-
from channels.handler import AsgiHandler
from channels import Channel, Group
from channels.sessions import channel_session
from channels.auth import http_session_user, channel_session_user, channel_session_user_from_http
import service.msg_agent as msg_agent
import multiprocessing, threading
import json, gc, sys, time

gc.enable()
gc.set_threshold(360, 8, 8)

# 全局缓存对象
# cache = {}
lock = multiprocessing.Lock()
usermap = dict()


def stop_process(user):
    try:
        if user in usermap:
            usermap[user]['thread_stop'] = True
            time.sleep(0.003)
            proc = usermap[user]['process']
            # if proc.is_alive():
            usermap[user]['queue'].close()
            usermap[user]['thread_queue'].close()
            proc.terminate()
            proc.join()
            time.sleep(0.002)
            del usermap[user]['queue']
Example #60
0
 def run():
     logger.info("Running")
     change_resource_limit(soft_file_limit)
     if gc_thresholds:
         gc.set_threshold(*gc_thresholds)
     run_command()