def disabled_test_collect_threads(self):
start = time.time()
def a(end):
while time.time() < end: pass
def thread1_func():
a(time.time() + 0.2)
def thread2_func():
a(time.time() + 0.3)
collector = Collector(interval=0.01, mode='prof')
collector.start()
thread1 = threading.Thread(target=thread1_func)
thread2 = threading.Thread(target=thread2_func)
thread1.start()
thread2.start()
a(time.time() + 0.1)
while thread1.isAlive(): pass
while thread2.isAlive(): pass
thread1.join()
thread2.join()
end = time.time()
collector.stop()
elapsed = end - start
self.assertTrue(0.3 < elapsed < 0.4, elapsed)
counts = self.filter_stacks(collector)
expected = {
('a', 'test_collect_threads'): 10,
('a', 'thread1_func'): 20,
('a', 'thread2_func'): 30,
}
self.check_counts(counts, expected)
def test_collector(self):
start = time.time()
def a(end):
while time.time() < end: pass
c(time.time() + 0.1)
def b(end):
while time.time() < end: pass
c(time.time() + 0.1)
def c(end):
while time.time() < end: pass
collector = Collector(interval=0.01, mode='prof')
collector.start()
a(time.time() + 0.1)
b(time.time() + 0.2)
c(time.time() + 0.3)
end = time.time()
collector.stop()
elapsed = end - start
self.assertTrue(0.8 < elapsed < 0.9, elapsed)
counts = self.filter_stacks(collector)
expected = {
('a', 'test_collector'): 10,
('c', 'a', 'test_collector'): 10,
('b', 'test_collector'): 20,
('c', 'b', 'test_collector'): 10,
('c', 'test_collector'): 30,
}
self.check_counts(counts, expected)
# cost depends on stack depth; for this tiny test I see 40-80usec
time_per_sample = float(collector.sample_time) / collector.samples_taken
self.assertTrue(time_per_sample < 0.000100, time_per_sample)
def prepare(self):
starlight.data.reset_statistics()
self.did_trigger_update = starlight.update.check_version()
super().prepare()
if self.get_argument("profile", None) and os.environ.get("ALLOW_PROFILING"):
self.collector = Collector()
self.collector.start()
def initialize(self):
self.session = self.application.my_settings.get("db_session")()
self.graph = Graph()
if self.get_argument("_profile", False):
self.perf_collector = Collector()
self.perf_trace_uuid = str(uuid4())
self.perf_collector.start()
else:
self.perf_collector = None
self.perf_trace_uuid = None
self._request_start_time = datetime.utcnow()
stats.incr("requests")
stats.incr("requests_{}".format(self.__class__.__name__))
def __call__(self, environ, start_response):
if not self.should_profile():
return self.application(environ, start_response)
collector = Collector()
collector.start()
start = time.time()
try:
return self.application(environ, start_response)
finally:
stop = time.time()
collector.stop()
try:
self.save_data(environ, start, stop, collector)
except Exception as e:
self.logger.exception(e)
def start_profile(callback):
ProfileHandler.waiters.append(callback)
if ProfileHandler.profile_running:
return
logging.info('starting profiler, will run for %ds' %
ProfileHandler.DURATION)
ProfileHandler.profile_running = True
collector = Collector()
collector.start(ProfileHandler.DURATION)
yield gen.Task(IOLoop.current().add_timeout,
datetime.timedelta(seconds=ProfileHandler.DURATION))
collector.stop()
data = repr(dict(collector.stack_counts))
for waiter in ProfileHandler.waiters:
waiter(data)
logging.info('finished profile collection')
ProfileHandler.profile_running = False
def initialize(self, *args: Any, **kwargs: Any) -> None:
self.graph = Graph()
self.session = self.settings["session"]() # type: Session
self.template_engine = self.settings[
"template_engine"] # type: FrontendTemplateEngine
self.plugins = get_plugin_proxy()
session_factory = SingletonSessionFactory(self.session)
self.usecase_factory = create_graph_usecase_factory(
settings(), self.plugins, session_factory)
if self.get_argument("_profile", False):
self.perf_collector = Collector()
self.perf_trace_uuid = str(uuid4()) # type: Optional[str]
self.perf_collector.start()
else:
self.perf_collector = None
self.perf_trace_uuid = None
self._request_start_time = datetime.utcnow()
def initialize(self, *args, **kwargs):
# type: (*Any, **Any) -> None
self.graph = Graph()
self.session = kwargs["session"]() # type: Session
self.template_env = kwargs["template_env"] # type: Environment
self.usecase_factory = kwargs[
"usecase_factory"] # type: UseCaseFactory
if self.get_argument("_profile", False):
self.perf_collector = Collector()
self.perf_trace_uuid = str(uuid4()) # type: Optional[str]
self.perf_collector.start()
else:
self.perf_collector = None
self.perf_trace_uuid = None
self._request_start_time = datetime.utcnow()
stats.log_rate("requests", 1)
stats.log_rate("requests_{}".format(self.__class__.__name__), 1)
def initialize(self, *args, **kwargs):
# type: (*Any, **Any) -> None
self.graph = Graph()
self.session = self.settings["session"]() # type: Session
self.template_engine = self.settings[
"template_engine"] # type: FrontendTemplateEngine
self.plugins = get_plugin_proxy()
session_factory = SingletonSessionFactory(self.session)
self.usecase_factory = create_graph_usecase_factory(
settings(), self.plugins, session_factory)
if self.get_argument("_profile", False):
self.perf_collector = Collector()
self.perf_trace_uuid = str(uuid4()) # type: Optional[str]
self.perf_collector.start()
else:
self.perf_collector = None
self.perf_trace_uuid = None
self._request_start_time = datetime.utcnow()
stats.log_rate("requests", 1)
stats.log_rate("requests_{}".format(self.__class__.__name__), 1)
logging.error("initialized")
def get(self):
self.collector = Collector()
self.collector.start()
IOLoop.instance().add_timeout(datetime.timedelta(seconds=60),
self.finish_profile)
def main():
"""
Set up environment for processing
"""
parser = argparse.ArgumentParser(description='X12 Validation')
parser.add_argument('--config-file',
'-c',
action='store',
dest="configfile",
default=None)
parser.add_argument('--log-file',
'-l',
action='store',
dest="logfile",
default=None)
parser.add_argument('--map-path',
'-m',
action='store',
dest="map_path",
default=None,
type=check_map_path_arg)
parser.add_argument('--verbose', '-v', action='count')
parser.add_argument('--debug', '-d', action='store_true')
parser.add_argument('--quiet', '-q', action='store_true')
parser.add_argument('--html', '-H', action='store_true')
parser.add_argument('--exclude-external-codes',
'-x',
action='append',
dest="exclude_external",
default=[],
help='External Code Names to ignore')
parser.add_argument('--charset',
'-s',
choices=('b', 'e'),
help='Specify X12 character set: b=basic, e=extended')
#parser.add_argument('--background', '-b', action='store_true')
#parser.add_argument('--test', '-t', action='store_true')
parser.add_argument('--profile',
action='store_true',
help='Profile the code with plop')
parser.add_argument('input_files', nargs='*')
args = parser.parse_args()
logger = logging.getLogger('pyx12')
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
stdout_hdlr = logging.StreamHandler()
stdout_hdlr.setFormatter(formatter)
logger.addHandler(stdout_hdlr)
logger.setLevel(logging.INFO)
param = params.params(args.configfile)
if args.debug:
logger.setLevel(logging.DEBUG)
param.set('debug', True)
if args.verbose and args.verbose > 0:
logger.setLevel(logging.DEBUG)
if args.quiet:
logger.setLevel(logging.ERROR)
fd_997 = None
fd_html = None
flag_997 = True
param.set('exclude_external_codes', ','.join(args.exclude_external))
if args.map_path:
param.set('map_path', args.map_path)
if args.logfile:
try:
hdlr = logging.FileHandler(args.logfile)
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
except IOError:
logger.exception('Could not open log file: %s' % (args.logfile))
for src_filename in args.input_files:
try:
if not os.path.isfile(src_filename):
logger.error('Could not open file "%s"' % (src_filename))
continue
#fd_src = open(src_filename, 'U')
if flag_997:
fd_997 = tempfile.TemporaryFile()
if args.html:
if os.path.splitext(src_filename)[1] == '.txt':
target_html = os.path.splitext(src_filename)[0] + '.html'
else:
target_html = src_filename + '.html'
fd_html = open(target_html, 'w')
if args.profile:
from plop.collector import Collector
p = Collector()
p.start()
if x12n_document.x12n_document(param=param,
src_file=src_filename,
fd_997=fd_997,
fd_html=fd_html,
fd_xmldoc=None,
map_path=args.map_path):
sys.stderr.write('%s: OK\n' % (src_filename))
else:
sys.stderr.write('%s: Failure\n' % (src_filename))
#import profile
#prof_str = 'pyx12.x12n_document.x12n_document(param, src_filename, ' \
# + 'fd_997, fd_html, None, None)'
#print prof_str
#print param
#profile.run(prof_str, 'pyx12.prof')
p.stop()
try:
pfile = os.path.splitext(
os.path.basename(src_filename))[0] + '.plop.out'
pfull = os.path.join(
os.path.expanduser('~/.plop.profiles'), pfile)
print(pfull)
with open(pfull, 'w') as fdp:
fdp.write(repr(dict(p.stack_counts)))
except Exception:
logger.exception('Failed to write profile data')
sys.stderr.write('%s: bad profile save\n' % (src_filename))
else:
if x12n_document.x12n_document(param=param,
src_file=src_filename,
fd_997=fd_997,
fd_html=fd_html,
fd_xmldoc=None,
map_path=args.map_path):
sys.stderr.write('%s: OK\n' % (src_filename))
else:
sys.stderr.write('%s: Failure\n' % (src_filename))
if flag_997 and fd_997.tell() != 0:
fd_997.seek(0)
if os.path.splitext(src_filename)[1] == '.txt':
target_997 = os.path.splitext(src_filename)[0] + '.997'
else:
target_997 = src_filename + '.997'
codecs.open(target_997, mode='w',
encoding='ascii').write(fd_997.read())
if fd_997:
fd_997.close()
if fd_html:
fd_html.close()
except IOError:
logger.exception('Could not open files')
return False
except KeyboardInterrupt:
print("\n[interrupt]")
return True
def get_and_run_plop_collector():
from plop.collector import Collector
collector = Collector()
collector.start()
return collector
def __init__(self, application, request, **kwargs):
super(PlopProfileHandler, self).__init__(application, request,
**kwargs)
self.interval = 60
self.output = '/tmp/plop.out'
self.collector = Collector()
def process_view(self, request, view_func, _, __):
'process a single view, adding the collector'
request.collector = Collector()
request.collector.start()
request.plop_filename = self.get_filename(view_func)
async def get(self):
self.collector = Collector()
self.collector.start()
return await self.finish_profile()
