def run(self, key, value=None): if value is not None: obj = list(filter(lambda o: str(o) == value, get_objects()))[0] if key == "graph": from objgraph import show_refs if value is None: p = self.console.parent show_refs(self.console if p is None else p, refcounts=True, max_depth=3) else: show_refs(obj, refcounts=True, max_depth=3) elif key == "growth": from objgraph import get_leaking_objects, show_most_common_types show_most_common_types(objects=get_leaking_objects()) elif key == "info": from psutil import Process p = Process(os.getpid()) print_formatted_text(p.memory_info()) elif key == "leaking": from objgraph import get_leaking_objects print_formatted_text(get_leaking_objects()) elif key == "objects": data = [["Object", "#References"]] for o in get_objects(): if isinstance(o, (Console, Module)): data.append([str(o), str(getrefcount(o))]) t = BorderlessTable(data, "Consoles/Modules") print_formatted_text(t.table) elif key == "refs": if value is not None: print_formatted_text(getrefcount(obj), ":") pprint(get_referrers(obj))
def memory_tracing(key_type: str = "lineno", limit: int = 15): """ Traces memory consumption and prints memory-usage statistics when leaving the context :param key_type: :param limit: :return: """ tracemalloc.start() print("--- Tracing memory... ---") try: # Do computation ... yield None finally: snapshot = tracemalloc.take_snapshot() # snapshot = snapshot.filter_traces(( # tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), # tracemalloc.Filter(False, "<unknown>"), # )) top_stats = snapshot.statistics(key_type) print("--- Memory usage statistics: ---") print("Top %s lines:" % limit) for index, stat in enumerate(top_stats[:limit], 1): frame = stat.traceback[0] # replace "/path/to/module/file.py" with "module/file.py" filename = os.sep.join(frame.filename.split(os.sep)[-2:]) print("#%s: %s:%s: %.1f KiB" % (index, filename, frame.lineno, stat.size / 1024)) line = linecache.getline(frame.filename, frame.lineno).strip() if line: print(' %s' % line) other = top_stats[limit:] if other: size = sum(stat.size for stat in other) print("%s other: %.1f KiB" % (len(other), size / 1024)) total = sum(stat.size for stat in top_stats) print("\nTotal allocated size: %.1f KiB" % (total / 1024)) # May also be useful: import objgraph print("\nTypes of most common instances:") objgraph.show_most_common_types(limit=limit) print("\nObjects that do not have any referents:") objgraph.get_leaking_objects() print("\nIncrease in peak object counts since last call:") objgraph.show_growth(limit=limit) print("\ntuple objects tracked by the garbage collector:") objgraph.by_type('tuple') print("\ndict objects tracked by the garbage collector:") objgraph.by_type('dict') print("--- End of memory tracing ---")
def objgraph(self, request): import objgraph limit = int(request.args.get('limit', 10)) types = objgraph.most_common_types(limit=limit, shortnames=False) leaking = objgraph.most_common_types( limit=limit, objects=objgraph.get_leaking_objects(), shortnames=False, ) # html only links limits = [ 'Number of items: ', Link('{}', request.path + '?limit={}', 10).html(), Link('{}', request.path + '?limit={}', 20).html(), Link('{}', request.path + '?limit={}', 50).html(), ] return info_response( request, 'Python Objects', Content( 'Python Objects for Worker pid {}'.format(os.getpid()), 'h1', ), Content(' '.join(limits), 'p', text=False, escape=False), Content('Most Common Objects', 'h2'), Table(types), Content('Leaking Objects (no referrer)', 'h2'), Table(leaking), )
def objgraph(self, request): import objgraph limit = int(request.args.get('limit', 10)) types = objgraph.most_common_types(limit=limit, shortnames=False) leaking = objgraph.most_common_types( limit=limit, objects=objgraph.get_leaking_objects(), shortnames=False, ) # html only links limits = [ 'Number of items: ', Link('{}', request.path + '?limit={}', 10).html(), Link('{}', request.path + '?limit={}', 20).html(), Link('{}', request.path + '?limit={}', 50).html(), ] return info_response( request.environ, 'Python Objects', Content( 'Python Objects for Worker pid {}'.format(os.getpid()), 'h1', ), Content(' '.join(limits), 'p', text=False, escape=False), Content('Most Common Objects', 'h2'), Table(types, id='types'), Content('Leaking Objects (no referrer)', 'h2'), Table(leaking, id='leaking'), )
def show_leaked_objects(): """ not very useful """ func_that_will_leak() gc.collect() roots = objgraph.get_leaking_objects() print('roots len = %s' % len(roots)) objgraph.show_most_common_types(objects=roots) objgraph.show_refs(roots[:3], refcounts=True, filename='roots.png')
def print_memory(i): global object_counts print("\n\n--------------------- MEMORY -------------------------\n") print("TOTAL OBJECTS\n") o = len(gc.get_objects()) print(o) object_counts.append(o) del o print("\n") print("GROWTH\n") objgraph.show_growth() print("\n") print("COMMON TYPES\n") objgraph.show_most_common_types() print("\n") print("LEAKING OBJECTS\n") roots = objgraph.get_leaking_objects() print("\n") log.info("ROOTS pre-collect : {}\n".format(len(roots))) print("COMMON TYPES IN ROOTS\n") objgraph.show_most_common_types(objects=roots) print("\n") objgraph.show_refs(roots[:3], refcounts=True, filename=TEST_PATH + '/roots_' + str(i) + '.png') print("\n") log.info("Garbage pre collect: " + str(len(gc.garbage))) gc.collect() log.info("Garbage post collect: " + str(len(gc.garbage))) print("\n") roots = objgraph.get_leaking_objects() log.info("ROOTS post-collect : {}".format(len(roots))) print("\n\n---------------------------------------------------\n")
def all_stocks(self): category = [] price_range = [] i = 0 for stock in self.stock_list: try: pred = self.predict_one(stock, 5) category.append(pred[0]) price_range.append( [pred[1][pred[0]] * 100, pred[1][pred[0] + 1] * 100]) print(self.device + ": " + str(i)) if i % 50 == 0: objgraph.get_leaking_objects() except (TypeError, ValueError) as e: category.append(0) price_range.append([0.0, 0.0]) print("Type Error for {}".format(stock)) print(e) i += 1 return category, price_range
def _dump_memory_impl(limit: int) -> Mapping[str, Any]: nb_collected = [gc.collect(generation) for generation in range(3)] return { 'nb_collected': nb_collected, 'most_common_types': objgraph.most_common_types(limit=limit, shortnames=False), 'leaking_objects': objgraph.most_common_types(limit=limit, shortnames=False, objects=objgraph.get_leaking_objects()) }
def __call__(self, environ, start_response): request = webob.Request(environ) if request.path_info.startswith(self.signature): query = request.GET.get('query') obj_type = request.GET.get('object_type') obj_address = request.GET.get('object_address') if obj_address: # Create a graph for the object leaking = [objgraph.at(int(obj_address))] filename = tempfile.mktemp(suffix='.png') objgraph.show_refs(leaking, filename=filename) output = open(filename, 'r').read() os.unlink(filename) start_response( '200 Ok', [('Content-Type', 'image/png')]) return output output = StringIO() leaking = objgraph.get_leaking_objects() formats.htmlHeader(output, self.signature, request.host_url, query) output.write('<h3>Memory usage</h3>') output.write('<table>' + ''.join( map(lambda info: MEMORY_LINE.format(*info), objgraph.most_common_types(10, leaking))) + '</table>') if obj_type: # Display detail about leaking objects of a given type. output.write('<h4>Memory detail for %s</h4>' % obj_type) output.write('<ul>') for obj in leaking: if type(obj).__name__ == obj_type: output.write(OBJECT_LINE.format( id(obj), cgi.escape(str(obj)))) output.write('</ul>') output.write('<h3>Timing</h3>') formats.htmlFormat(output, query=query) start_response( '200 Ok', [('Content-Type', 'text/html; charset=UTF8')]) return output.getvalue() scale = self.find(request.path_info) scale.requests.mark() with scale.latency.time(): response = request.get_response(self.app) result = scale.statuses(response.status_int) if result is not None: result.mark() start_response( response.status, [a for a in response.headers.iteritems()]) return response.app_iter
def on_selection_change(self, *args): #pass if self.selection: #from listscreen import ListScreenItem objgraph.show_growth() print '...' roots = objgraph.get_leaking_objects() objgraph.show_most_common_types(objects=roots) print '...' objgraph.show_refs(roots[:3], refcounts=True, filename='sad.png') #objgraph.show_chain(objgraph.find_backref_chain(self.selection[0].__self__, objgraph.is_proper_module),filename='chain.png') #objgraph.show_backrefs(self.selection[0].__self__, filename='sample-backref-graph.png') print '...'
def _status_leak(self): if not self.leak: return pid = os.getpid() print("%d active threads" % threading.activeCount()) if guppy: print(self.heap.heap()) if psutil: process = psutil.Process(pid) if hasattr(process, "open_files"): print(process.open_files()) if hasattr(process, "connections"): print(process.connections()) if hasattr(process, "num_fds"): print(process.num_fds()) if hasattr(process, "memory_info_ex"): print(process.memory_info_ex()) if hasattr(process, "memory_maps"): print(process.memory_maps()) if objgraph: print("%d objects leaking" % len(objgraph.get_leaking_objects()))
def __dbg_mem(strn): if __DBGMEM: import gc, objgraph print print '#' * 80 print '#' * 80 print '##', strn print 'Collect', gc.collect() print 'Collect', gc.collect() roots = objgraph.get_leaking_objects() if roots: print len(roots) objgraph.show_most_common_types(objects=roots) objgraph.show_refs(roots[:3], refcounts=True, filename='tmp/%s.png' % strn.lower()) else: print 'Nothing' print '#' * 80 print '#' * 80 print
def prepareDocuments(docs): import pickle print("preparing %d documents" % len(docs)) chunksize = 15000 if functionCollection is not None: for i in range(0, len(docs), chunksize): chunk = docs[i:i + chunksize] functionCollection.moveToMemory(chunk, neededDocumentFunctions) #print("forget unnecessary document functions...") gc.collect() if config.debug_memory: print("garbage: ", len(gc.garbage)) print("15 most common types:") objgraph.show_most_common_types(limit=15) c_syntax_tree.showCMemoryStatistics() showMemoryStatistics() functionCollection.showMemoryStatistics() functionCollection.getFunction( features.stanfordTreeDocumentFunction ).cachedValues.showMemoryStatistics() print("leaking: ", len(objgraph.get_leaking_objects()))
grbindx=pygrib.index(grib_file, 'indicatorOfParameter', 'indicatorOfTypeOfLevel', 'level') #plottypes.windplot(grbindx, source, map_types, outdir = outdir) plottypes.temp_leak_plot(grbindx, source, map_types, outdir = outdir) print 'SELF ',resource.getrusage(resource.RUSAGE_SELF).ru_maxrss print 'CHILDREN', resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss #plottypes.cloud_plot(grbindx, source, map_types, outdir = outdir) #plottypes.prec_plot(grbindx, grbindx_prev, source, map_types, outdir = outdir) #sys.exit(1) # one file done if grbindx_prev is not None: grbindx_prev.close() # close previous(?) grbindx_prev = grbindx # def new previous #grbindx.close() #for k,v in globals().items(): print 'globals:', k,'=',v #sys.exit(1) # all files done #sys.exit(1) # all sources done gc.collect() import objgraph objgraph.get_leaking_objects()
def discover(A, system_params, initial_state_set, budget=None): final_state_set = set() Q = queue.Queue(maxsize=0) examined_state_set = set() # initialize the Q with initial states # ##!!##logger.debug('Adding initial states to Q') for init_state in initial_state_set: Q.put(init_state) if settings.MEMLEAK_TEST: objgraph.show_growth(shortnames=False) U.pause() while not Q.empty(): #objgraph.show_growth(shortnames=False) abs_state = Q.get(False) # ##!!##logger.debug('{} = Q.get()'.format(abs_state)) if not (A.is_terminal(abs_state) or abs_state in examined_state_set or not A.in_ROI(abs_state)): # ##!!##logger.debug('decided to process abs_state') # Mark it as examined examined_state_set.add(abs_state) # Find all reachable abstract states using simulations abs2rch_abs_state_dict = get_reachable_abs_states(A, system_params, [abs_state]) # add the new reached states only if they have not been # processed before # ##!!##logger.debug('abs2rch_abs_state_dict.values()\n{}'.format(abs2rch_abs_state_dict.values())) rchd_abs_state_set = abs2rch_abs_state_dict[abs_state] # TODO: abstract away the graph maybe?? # Hide the call behind add_relation(A1, A2) for rchd_abs_state in rchd_abs_state_set: # ##!!##logger.debug('reached abstract state {}'.format(rchd_abs_state)) # query if the reached state is a final state or not? # If yes, tag it so if system_params.is_final(A, rchd_abs_state): final_state_set.add(rchd_abs_state) else: # print('found a final state') # exit() Q.put(rchd_abs_state, False) # moving below to the abstraction itself # A.add_relation(abs_state, rchd_abs_state) # A.G.add_edge(abs_state, rchd_abs_state) # n = self.get_n_for(abs_state) + 1 # self.set_n_for(rchd_abs_state, n) # end while loop # ##!!##logger.debug('Abstraction discovery done') # ##!!##logger.debug('Printing Abstraction\n {}'.format(str(A))) if settings.MEMLEAK_TEST: objgraph.show_growth(shortnames=False) from core.PACell import PlantAbstractState_ as PS # print('PlantAbstractState.instance_ctr = ', PS.instance_ctr) # print('PlantAbstractState.list_size = ', len(PS.instance_store)) # print('PlantAbstractState.set_size = ', len(PS.instance_set)) # objgraph.show_backrefs( # PS.instance_store[0], max_depth=10, too_many=5, # highlight=None, filename='objgraph1.png', # extra_info=None, refcounts=False, shortnames=False) from core.abstraction import AbstractState_ as AS print('num instances of plant states:', PS.instance_ctr) print('num instances of abstract states:', AS.instance_ctr) print('memory leaks below...') objgraph.get_leaking_objects() print('='*20) U.pause() return final_state_set
def func_wrapper(): pp.pprint(objgraph.get_leaking_objects()) abort()
def objgraph_leaking_objects(self, args): """ This is supposed to count reference counting bugs in C-level, see https://mg.pov.lt/objgraph/#reference-counting-bugs Example: $ fuglu_control objgraph_leaking_objects '{"nresults": 5}' ---------------- Leaking objects: ---------------- params: * nresults: 5 * lowercase: True * dont_startwith: * must_startwith: * dont_contain: * must_contain: builtins.dict : 797 builtins.list : 132 builtins.tuple : 28 builtins.method : 13 builtins.weakref : 12 """ res = u"----------------\n" \ + u"Leaking objects:\n" \ + u"----------------\n\n" if OBJGRAPH_EXTENSION_ENABLED: defaults = { "nresults": 20, "lowercase": True, "dont_startwith": ["builtins", "_"], "dont_contain": [], "must_startwith": [], "must_contain": [] } if not args: args = {} # fill filter lists and other vars from dict res, inputdict = ControlSession.prepare_objectgraph_list_from_dict( args, res, defaults) roots = None types_list = None finalfilter = None try: roots = objgraph.get_leaking_objects() # build filter finalfilter = ControlSession.buildfilter( dont_contain=inputdict["dont_contain"], dont_startwith=inputdict["dont_startwith"], must_contain=inputdict["must_contain"], must_startwith=inputdict["must_startwith"], lowercase=inputdict["lowercase"]) types_list = objgraph.most_common_types( objects=roots, limit=inputdict["nresults"], shortnames=False, filter=finalfilter) for otype in types_list: res += u"%s : %u\n" % otype except Exception as e: res += force_uString(e) self.logger.exception(e) finally: if roots: del roots if types_list: del types_list if finalfilter: del finalfilter else: res = u"please install module 'objgraph'" return res
def _dump_memory_impl( limit: int, analyze_type: Optional[str], python_internals_map: bool = False, ) -> Mapping[str, Any]: nb_collected = [gc.collect(generation) for generation in range(3)] result = { "nb_collected": nb_collected, "most_common_types": objgraph.most_common_types(limit=limit, shortnames=False), "leaking_objects": objgraph.most_common_types(limit=limit, shortnames=False, objects=objgraph.get_leaking_objects()), } if python_internals_map and not analyze_type: analyze_type = "builtins.dict" if analyze_type: # timeout after one minute, must be set to a bit less that the timeout of the broadcast in _views.py timeout = time.monotonic() + 60 mod_counts: Dict[str, int] = {} biggest_objects: List[Tuple[float, Any]] = [] result[analyze_type] = {} for obj in objgraph.by_type(analyze_type): if analyze_type == "builtins.function": short = obj.__module__.split( ".")[0] if obj.__module__ is not None else "" mod_counts[short] = mod_counts.get(short, 0) + 1 else: if analyze_type == "builtins.dict": python_internal = False if not (FILES_FIELDS - set(obj.keys())): python_internal = True if (not ({"scope", "module", "locals", "globals"} - set(obj.keys())) and isinstance(obj["globals"], dict) and not (FILES_FIELDS - set(obj["globals"].keys()))): python_internal = True if (python_internal and not python_internals_map or not python_internal and python_internals_map): continue size = get_size(obj) / 1024 if len(biggest_objects ) < limit or size > biggest_objects[0][0]: biggest_objects.append((size, obj)) biggest_objects.sort(key=lambda x: x[0]) if len(biggest_objects) > limit: biggest_objects = biggest_objects[-limit:] if time.monotonic() > timeout: result[analyze_type]["timeout"] = True break if analyze_type == "builtins.function": result[analyze_type]["modules"] = [ dict(module=i[0], nb_func=i[1]) for i in sorted(mod_counts.items(), key=lambda x: -x[1])[:limit] ] elif analyze_type == "linecache": import linecache cache = linecache.cache result[analyze_type]["biggest_objects"] = sorted( (dict(filename=k, size_kb=get_size(v)) for k, v in cache.items()), key=lambda i: -(cast(int, i["size_kb"])), ) else: biggest_objects.reverse() result[analyze_type]["biggest_objects"] = [ dict(size_kb=i[0], repr=repr(i[1])) for i in biggest_objects ] return result
for device in all_devices: #do pumping if there are items in coming list if device.coming: is_need_to_pump = True device.pump() assert test_dict['packet'] == packet.__dict__ assert test_dict['recipient'] == machines[j] l1 = Letter( title='First email', sender='*****@*****.**', recipient='*****@*****.**', text= 'c374hfvc39pbvnc 7hv475v4bv vh549g7h3pabv 3fg37b30v 3v3p4bgf3gv 3vb54804v 459!', date=datetime.now().strftime("%d-%m-%Y/%H:%M")) l1bytes = bytes(str(l1), 'utf-8') l2 = Letter( title='First email', sender='*****@*****.**', recipient='*****@*****.**', text= 'c374hfvc39pbvnc 7hv475v4bv vh549g7h3pabv 3fg37b30v 3v3p4bgf3gv 3vb54804v 459!', date=datetime.now().strftime("%d-%m-%Y/%H:%M")) l2bytes = bytes(str(l2), 'utf-8') roots = objgraph.get_leaking_objects() print(len(roots)) objgraph.show_most_common_types(objects=roots)
def get_leaking_objects(file=None, limit=5): roots = objgraph.get_leaking_objects() objgraph.show_refs(roots[:limit], refcounts=True, output=file)
def show_leak_obj(self): root = objgraph.get_leaking_objects() logging.error("leak object: {}".format(len(root))) import pdb; pdb.set_trace()
def add_single_experiment(directory, df_filepath, datasets): csv_filenames = [ f for f in os.listdir(directory) if f[0:5] == "resul" and f[-4:] == ".csv" ] print(list(os.walk(directory))) weights_folder = [f for f in os.walk(directory)][0][1][0] for filename in csv_filenames: experiment = directory_to_experiment_info(directory) with open("./" + directory + "/" + filename, "r") as csvfile: logging.debug("./" + directory + "/" + filename) plots = csv.reader(csvfile, delimiter=",") headers = next(plots, None) for header in headers: experiment[header] = [] for row in plots: for i, header in enumerate(headers): experiment[header].append(float(row[i])) experiment["minimum_val_loss"] = min(experiment["val_loss"]) experiment["minimum_loss"] = min(experiment["loss"]) number = int(filename.split("-")[-1].split(".")[0]) experiment["number"] = number experiment["repeat"] = math.floor(number / 4) if (number % 4) / 2 < 1: experiment["elastic_distortions"] = True else: experiment["elastic_distortions"] = False if (number % 4) % 2 != 0: experiment["separate_channel_ops"] = True else: experiment["separate_channel_ops"] = False print(experiment.keys()) if not experiment_in_dataframe(df_filepath, experiment): weights_file = (directory + "/" + weights_folder + "/" + "mask_rcnn_fk2018_best.h5") config = FK2018.FKConfig() print("got config") class InferenceConfig(config.__class__): # Run detection on one image at a time GPU_COUNT = 1 IMAGES_PER_GPU = 1 config = InferenceConfig() config.display() # Device to load the neural network on. # Useful if you're training a model on the same # machine, in which case use CPU and leave the # GPU for training. # DEVICE = "/gpu:0" # /cpu:0 or /gpu:0 # Inspect the model in training or inference modes # values: 'inference' or 'training' # TODO: code for 'training' test mode not ready yet TEST_MODE = "inference" # Must call before using the dataset # with tf.device(DEVICE): tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with tf.Session(config=tf_config).as_default(): model = modellib.MaskRCNN(mode="inference", model_dir="./log", config=config) # Load weights logging.debug("Loading weights " + str(weights_file)) model.load_weights(weights_file, by_name=True) image_ids = datasets[0].image_ids tracker = SummaryTracker() logging.debug("getting stats") logging.debug("tunasand stats") experiment["AP_list"], \ experiment["classless_AP_list"], \ experiment["precision_list"], \ experiment["classless_precision_list"], \ experiment["recall_list"], \ experiment["classless_recall_list"], \ experiment["predicted_class_list"], \ experiment["gt_class_list"], \ experiment["predicted_size_list"], \ experiment["gt_size_list"], \ experiment["overlaps"], \ experiment["classless_overlaps_list"], \ experiment["total_predicted_pixels"], \ experiment["total_groundtruth_pixels"], \ experiment["total_overlapping_pixels"] = compute_both_batch_aps(image_ids, datasets[0], model, config) #get_stats(weights_file, datasets[0]) objgraph.show_most_common_types() roots = objgraph.get_leaking_objects() print(len(roots)) tracker.print_diff() for i, dataset in enumerate( ["AE_area1", "AE_area2", "AE_area3"]): image_ids = datasets[i + 1].image_ids logging.debug(f"aestats, {dataset}") experiment[f"AP_list_{dataset}"], \ experiment[f"classless_AP_list_{dataset}"], \ experiment[f"precision_list_{dataset}"], \ experiment[f"classless_precision_list_{dataset}"], \ experiment[f"recall_list_{dataset}"], \ experiment[f"classless_recall_list_{dataset}"], \ experiment[f"predicted_class_list_{dataset}"], \ experiment[f"gt_class_list_{dataset}"], \ experiment[f"predicted_size_list_{dataset}"], \ experiment[f"gt_size_list_{dataset}"], \ experiment[f"overlaps_{dataset}"], \ experiment[f"classless_overlaps_list_{dataset}"], \ experiment[f"total_predicted_pixels_{dataset}"], \ experiment[f"total_groundtruth_pixels_{dataset}"], \ experiment[f"total_overlapping_pixels_{dataset}"] = compute_both_batch_aps(image_ids, datasets[i+1], model, config) #get_stats(weights_file, datasets[i+1]) objgraph.show_growth() roots = objgraph.get_leaking_objects() print(len(roots)) tracker.print_diff() update_dataframe(df_filepath, experiment) else: print("already in dataframe, skipping " + filename)
def main(args): memory_used = [] process = psutil.Process(os.getpid()) #TODO add into argparser IS_EAST_IMAGE_TEST = True NUM_ARRAYS_PER_FILE = 10000 #TODO decode function needs this value as part of dataset map function, hence for now harcoded value # if needed chnage manually at func `numpy_array_decode` in dummy_dataset.py also NUM_FEATURES = 250 NUM_IMAGES_PER_FILE = 8 BATCH_SIZE = 4 TRAIN_DATA = os.getcwd() + "/data/train_data_img" VAL_DATA = os.getcwd() + "/data/val_data_img" MODEL_DIR = os.getcwd() + "/data/" + "east_net" EXPORT_DIR = MODEL_DIR + "/" + "export" NUM_EPOCHS = 3 NUM_SAMPLES_PER_FILE = NUM_IMAGES_PER_FILE if args["dataset"] == "numpy": IS_EAST_IMAGE_TEST = False BATCH_SIZE = 128 TRAIN_DATA = os.getcwd() + "/data/train_data" VAL_DATA = os.getcwd() + "/data/val_data" MODEL_DIR = os.getcwd() + "/" + "data/fwd_nnet" EXPORT_DIR = MODEL_DIR + "/" + "export" NUM_EPOCHS = 3 NUM_SAMPLES_PER_FILE = NUM_ARRAYS_PER_FILE elif args["dataset"] == "east": pass else: print_error("Invalid dataset") TOTAL_STEPS_PER_FILE = NUM_SAMPLES_PER_FILE / BATCH_SIZE if args["delete"] == True: print_info("Deleting old data files") shutil.rmtree(TRAIN_DATA) shutil.rmtree(VAL_DATA) gen_data(IS_EAST_IMAGE_TEST=IS_EAST_IMAGE_TEST, TRAIN_DATA=TRAIN_DATA, VAL_DATA=VAL_DATA, NUM_SAMPLES_PER_FILE=NUM_SAMPLES_PER_FILE, NUM_FEATURES=NUM_FEATURES, number_files=int(args["num_tfrecord_files"])) if args["mode"] == "test_iterator": print('objgraph growth list start') objgraph.show_growth(limit=50) print('objgraph growth list end') test_dataset(data_path=TRAIN_DATA, BATCH_SIZE=BATCH_SIZE, IS_EAST_IMAGE_TEST=IS_EAST_IMAGE_TEST) test_dataset(data_path=TRAIN_DATA, BATCH_SIZE=BATCH_SIZE, IS_EAST_IMAGE_TEST=IS_EAST_IMAGE_TEST) test_dataset(data_path=VAL_DATA, BATCH_SIZE=BATCH_SIZE, IS_EAST_IMAGE_TEST=IS_EAST_IMAGE_TEST) print('objgraph growth list start') objgraph.show_growth(limit=50) print('objgraph growth list end') return # print(dataset_to_iterator(data_path=TRAIN_DATA)) if IS_EAST_IMAGE_TEST: model = EASTTFModel(model_root_directory="store") else: model = NNet() estimator = tf.estimator.Estimator( model_fn=model, config=_init_tf_config(TOTAL_STEPS_PER_FILE=TOTAL_STEPS_PER_FILE, MODEL_DIR=MODEL_DIR), params=None) memory_usage_psutil() print('objgraph growth list start') objgraph.show_growth(limit=50) print('objgraph growth list end') # print(objgraph.get_leaking_objects()) # for epoch in tqdm(range(NUM_EPOCHS)): print("\n\n\n\n\n\n") print_error(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> New Epoch") memory_usage_psutil() # memory_used.append(process.memory_info()[0] / float(2 ** 20)) print_error(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Training") # train(estimator=estimator, # TRAIN_DATA=TRAIN_DATA, # BATCH_SIZE=BATCH_SIZE, # IS_EAST_IMAGE_TEST=IS_EAST_IMAGE_TEST) # print_error(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Evaluating") # evaluate(estimator=estimator, # VAL_DATA=VAL_DATA, # BATCH_SIZE=BATCH_SIZE, # IS_EAST_IMAGE_TEST=IS_EAST_IMAGE_TEST) train_n_evaluate(estimator=estimator, TRAIN_DATA=TRAIN_DATA, VAL_DATA=VAL_DATA, BATCH_SIZE=BATCH_SIZE, IS_EAST_IMAGE_TEST=IS_EAST_IMAGE_TEST, max_steps=None, NUM_EPOCHS=NUM_EPOCHS) print('objgraph growth list start') objgraph.show_growth(limit=50) print('objgraph growth list end') memory_usage_psutil() # plt.plot(memory_used) # plt.title('Evolution of memory') # plt.xlabel('iteration') # plt.ylabel('memory used (MB)') # plt.savefig("logs/" + args["dataset"] + "_dataset_memory_usage.png") # plt.show() print_error(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> New Epoch") export_model(estimator=estimator, model_export_path=EXPORT_DIR, IS_EAST_MODEL=IS_EAST_IMAGE_TEST) (objgraph.get_leaking_objects())
def do_update(self): leaked = objgraph.get_leaking_objects() self.g.update() leaked = objgraph.get_leaking_objects() ts = objgraph.typestats(leaked) self.assertFalse(ts)