def convert_loss_scale_api(node): """Convert loss scale related Tensorflow APIs""" if isinstance(node.func, ast.Attribute): if node.func.attr == "FixedLossScale": log_msg( getattr(node, 'lineno', 'None'), "change tf.train.experimental.FixedLossScale" " to FixedLossScaleManager") node.func = ast.Name(id="FixedLossScaleManager", ctx=ast.Load()) if len(node.keywords) == 1: node.keywords[0].arg = "loss_scale" util_global.set_value('need_conver', True) return node if node.func.attr == "DynamicLossScale": return convert_dynamic_loss_scale(node) if node.func.attr == "MixedPrecisionLossScaleOptimizer": log_msg( getattr(node, 'lineno', 'None'), "change tf.train.experimental.MixedPrecisionLossScaleOptimizer" " to NPULossScaleOptimizer") node.func = ast.Name(id="NPULossScaleOptimizer", ctx=ast.Load()) for keyword in node.keywords: if keyword.arg == "loss_scale": keyword.arg = "loss_scale_manager" if (len(util_global.get_value("distributed_mode", "")) != 0): node.keywords.append( ast.keyword(arg="is_distributed", value=pasta.parse("True"))) util_global.set_value('need_conver', True) return node
def import_from(node): """Modify node based on import module""" if node.module: values = node.module.split(".") if "keras" in values: util_global.set_value('is_keras_net', True) if "horovod" in values: log_msg(getattr(node, "lineno", "None"), "remove horovod import line to None") util_global.set_value('has_hvd_api', True) new_node = ast.Expr(value=ast.NameConstant(value=None)) ast.copy_location(new_node, node) util_global.set_value('need_conver', True) return new_node for value in node.names: if isinstance(value, ast.alias): values = value.name.split(".") if "keras" in values: util_global.set_value('is_keras_net', True) if "horovod" in values: log_msg(getattr(node, "lineno", "None"), "remove horovod import line to None") util_global.set_value('has_hvd_api', True) new_node = ast.Expr(value=ast.NameConstant(value=None)) ast.copy_location(new_node, node) util_global.set_value('need_conver', True) return new_node util_global.set_value('need_conver', True) return node
def ast_if(node): if isinstance(node.test, ast.Compare): if len(node.test.comparators) == 1 and isinstance( node.test.comparators[0], ast.Str): if node.test.comparators[0].s == "__main__": util_global.set_value("is_main_file", False) util_global.set_value("has_main_func", True) if util_global.get_value("is_keras_net", False): log_msg(getattr(node, "lineno", "None"), " add keras session npu config") close_sess_call = ast.Call( func=ast.Name(id="close_session", ctx=ast.Load()), args=[ast.Name(id="npu_keras_sess", ctx=ast.Load())], keywords=[]) keras_sess_assign = ast.Assign( targets=[ ast.Name(id="npu_keras_sess", ctx=ast.Store()) ], value=ast.Call(func=ast.Name( id="set_keras_session_npu_config", ctx=ast.Load()), args=[], keywords=[])) node.body = [keras_sess_assign] + node.body + [ ast.Expr(value=close_sess_call) ] util_global.set_value('need_conver', True) if util_global.get_value("has_hccl_api", False): log_msg(getattr(node, "lineno", "None"), " add npu resource init api") close_sess_call = ast.Call( func=ast.Name(id="close_session", ctx=ast.Load()), args=[ast.Name(id="npu_sess", ctx=ast.Load())], keywords=[]) init_assign = ast.Assign(targets=[ ast.Tuple(elts=[ ast.Name(id="npu_sess", ctx=ast.Store()), ast.Name(id="npu_shutdown", ctx=ast.Store()) ], ctx=ast.Store()) ], value=ast.Call(func=ast.Name( id="init_resource", ctx=ast.Load()), args=[], keywords=[])) shutdown_call = ast.Call(func=ast.Name( id="shutdown_resource", ctx=ast.Load()), args=[ ast.Name(id="npu_sess", ctx=ast.Load()), ast.Name(id="npu_shutdown", ctx=ast.Load()) ], keywords=[]) node.body = [init_assign] + node.body + [ ast.Expr(value=shutdown_call), ast.Expr(value=close_sess_call) ] util_global.set_value('need_conver', True) return node
def imge_transfer(url,surl): # post url to image downloader pts = url.split("?")[0].strip("/").split("/") name = "spider/%s" % "/".join(pts[2:]) nsrc = url_temp % name util.log_msg("image_process\t%s\t%s\t%s" % (url,name,surl)) return (url,"/%s/%s" % (bucket_name,name),nsrc)
def create_post(post, msg, post_type, msg_type): """ function for posting to subreddit; separate post for long monster table posts; self-edits signature for easy delete request PM link """ try: for i, m in enumerate(msg): if len(msg) > 1: m = m.replace('___MTABLE___', "(%s of %s)" % (i + 1, len(msg))) else: m = m.replace('___MTABLE___', "") if post_type == 'SUBMISSIONS': if debug: print m + signature_intro + signature else: c = post.add_comment(m + signature_intro + signature) log_msg("Made a %s comment in %s" % (msg_type, post.short_link)) update_db(log_coll, stat_coll, msg_type, post.short_link, '') elif post_type == 'COMMENTS': if debug: print m + signature_intro + signature else: c = post.reply(m + signature_intro + signature) log_msg("Made a %s reply to %s" % (msg_type, post.permalink)) update_db(log_coll, stat_coll, msg_type, post.permalink, '') sig_temp = signature_add.replace('___CID___', str(c.id)) sig_temp = sig_temp.replace('___PID___', str(c.link_id)[3:]) if not debug: sleep(SLEEP) m_tmp = c.body.replace('^^Processing...', sig_temp) m_tmp = m_tmp.replace('&#', '&#') r.get_info(thing_id='t1_' + str(c.id)).edit(m_tmp) sleep(SLEEP_LONG) except Exception as e: log_error(e)
def get_info(content,info=None): try: p = {"collection_token":"","cursor":"","disablepager":False,"overview":False,"profile_id":"100008346345446","pagelet_token":"","tab_key":"friends","lst":"","ftid":None,"order":None,"sk":"friends","importer_state":None} loader_info = re.findall('"enableContentLoader"([^\[]+\[[^\]]+){2}',content) lst = None pglt = None if info is None: page_token = re.findall('pagelet_token"?:[^\}]+}',content) page_token = page_token[0] lst = page_token.split("\"")[-2] pglt = page_token[page_token.find(":\"")+2:].split("\"")[0] else: lst = info[0] pglt = info[1] loader_info = loader_info[0] pfx1 = "pagelet_timeline_app_collection_" idx1 = loader_info.find(pfx1) ct = None if idx1 > -1: ct = loader_info[idx1+len(pfx1):].split("\"")[0] cs = loader_info.split("\"")[-2] np = p.copy() np["collection_token"] = ct np["cursor"] = cs np["pagelet_token"] = pglt np["lst"] = lst np = json.dumps(np) return (np,(lst,pglt)) except Exception: tp, e,trace = sys.exc_info() util.log_msg("parse paramerter error:"+str(e),"error") return (None,None)
def convert_dynamic_loss_scale(node): """Convert dynamic loss scale related Tensorflow APIs""" log_msg( getattr(node, 'lineno', 'None'), "change tf.train.experimental.DynamicLossScale" " to ExponentialUpdateLossScaleManager") node.func = ast.Name(id="ExponentialUpdateLossScaleManager", ctx=ast.Load()) def check_arg(node): initial_loss_scale = None increment_period = None multiplier = None for index, arg in enumerate(node.args): if index == 0: initial_loss_scale = arg if index == 1: increment_period = arg if index == 2: multiplier = arg for keyword in node.keywords: if keyword.arg == "initial_loss_scale": keyword.arg = "init_loss_scale" initial_loss_scale = keyword if keyword.arg == "increment_period": keyword.arg = "incr_every_n_steps" increment_period = keyword if keyword.arg == "multiplier": keyword.arg = "incr_ratio" multiplier = keyword return (initial_loss_scale, increment_period, multiplier) (initial_loss_scale, increment_period, multiplier) = check_arg(node) if initial_loss_scale: if not isinstance(initial_loss_scale, ast.keyword): node.keywords.append( ast.keyword(arg="init_loss_scale", value=initial_loss_scale)) else: node.keywords.append( ast.keyword(arg="init_loss_scale", value=pasta.parse("2**15"))) if increment_period: if not isinstance(increment_period, ast.keyword): node.keywords.append( ast.keyword(arg="incr_every_n_steps", value=increment_period)) else: node.keywords.append( ast.keyword(arg="incr_every_n_steps", value=pasta.parse("2000"))) if multiplier: if not isinstance(multiplier, ast.keyword): node.keywords.append( ast.keyword(arg="incr_ratio", value=multiplier)) else: node.keywords.append( ast.keyword(arg="incr_ratio", value=pasta.parse("2"))) node.args = [] util_global.set_value('need_conver', True) return node
def get_params(p): try: status,op = subprocess.getstatusoutput("node "+p) if status == 0: return op else: return None except Exception: tp, e,trace = sys.exc_info() util.log_msg("execute javascript file error:%s" % str(e)) return None
def get_user_info(u,fp): ourl = "http://www.toutiao.com/c/user/%s/" % u url,html = util.get_page({"url":ourl}) p,h = get_urlinfo(url) params = get_params("func.js") uid = get_userinfo(html) if params is not None and uid is not None: params = json.loads(params) params["user_id"] = uid path = "/c/user/article/" nurl = "%s//%s%s" % (p,h,path) count = 3 while True: url,html = util.get_page({"url":nurl,"data":params,"method":"post"}) if html is None or len(html) == 0: util.log_msg("could not get data from url:%s,data:%s,uid:%s" % (nurl,str(params),u)) break mp = json.loads(html) if "data" in mp and isinstance(mp["data"],list): if len(mp["data"]) == 0: util.log_msg("no data from response.url:%s" % nurl) result = [] for item in mp["data"]: turl = util.parse_url(url,item["source_url"]) try: get_article(turl,url,item,fp,result) except Exception: tp, e,trace = sys.exc_info() util.log_msg("get article(url:%s) info error:%s" % (turl,str(e))) if len(result) > 0: if fp is None: insert_into_db(result) else: for item in result: fp.write("[%s]\t%s\t%s\n" % (time.ctime(),u,json.dumps(item))) else: util.log_msg("no data in content.url:%s" % nurl) if mp["has_more"]: params = get_params("func.js") params = json.loads(params) params["user_id"] = uid nxt = mp["next"] for key in nxt.keys(): params[key]=nxt[key] else: break count -= 1 if count <= 0: break else: util.log_msg("could not parse data from html file,need to check this out.url:%s,referer:%s." % (ourl,referer))
def delete_post(post, type, parent_url, user): """ function for deleting own post """ try: if type == 'SCORE': log_msg("Deleting post under %s due to downvote" % parent_url) update_db(log_coll, stat_coll, 'DEL_SCORE', parent_url, user) if type == 'PM': log_msg("Deleting post under %s due to PM request from %s" % (parent_url, user)) update_db(log_coll, stat_coll, 'DEL_PM', parent_url, user) post.delete() sleep(SLEEP) except Exception as e: log_error(e)
def table_output(padx, msg): """ uses PADX data to build reddit post output; automatically creates a new post if the # of chars > 10000 (reddit limit); automatically apends bot signature """ table_header = "%s###### 	\n#### 	\n##### 	\n|||Expanded Monster Information ___MTABLE___ [hover to view]|\n:--:|--:|:--\n" % signature_intro if len(msg) == 0: msg.append(table_header) i = len(msg) - 1 sid = int(padx.id) / 200 cid = int(padx.id) % 200 msg_temp = "[](#I/S%02d/C%03d/%s \"%s\")|" % (sid, cid, padx.id, padx.name) msg_temp = "%s#%s|**[%s](http://www.puzzledragonx.com/en/monster.asp?n=%s)**\n" % (msg_temp, padx.id, padx.name, padx.id) if len(padx.type_formatted) > 0: msg_temp = "%s%s" % (msg_temp, padx.type_formatted[0]) else: msg_temp = "%s " % msg_temp if padx.ls_id != '0': msg_temp = "%s|Leader|**[%s](http://www.puzzledragonx.com/en/leaderskill.asp?s=%s)**: %s\n" % (msg_temp, padx.ls_name, padx.ls_id, padx.ls_text) else: msg_temp = "%s|Leader|**%s**\n" % (msg_temp, padx.ls_name) if len(padx.type_formatted) > 1: msg_temp = "%s%s" % (msg_temp, padx.type_formatted[1]) else: msg_temp = "%s " % msg_temp if padx.as_id != '0': msg_temp = "%s |Active|**[%s](http://www.puzzledragonx.com/en/skill.asp?s=%s)**: %s (%s)\n" % (msg_temp, padx.as_name, padx.as_id, padx.as_text, padx.acd_text) else: msg_temp = "%s |Active|**%s**\n" % (msg_temp, padx.as_name) if len(padx.awk) != 0: awk_temp = "" for a in padx.awk: awk_temp = "%s[](#I/misc/C%03d/) " % (awk_temp, a) msg_temp = "%s |Awoken|%s\n" % (msg_temp, awk_temp) else: msg_temp = "%s |Awoken|None\n" % (msg_temp) if len(msg[i]) + len(msg_temp) + len(signature) + len(signature_add) > MSG_LEN_LIMIT: log_msg("message long... splitting") update_db(log_coll, stat_coll, 'SPLIT', '', '') msg.append(table_header) msg[i + 1] = msg[i + 1] + msg_temp else: msg[i] = msg[i] + msg_temp return msg
def insert_keras_dropout_import(r_node): """Add keras dropout import module""" npu_alias = ast.alias(name='npu_convert_dropout', asname=None) npu_import = ast.ImportFrom(module='npu_bridge.estimator.npu', names=[npu_alias], level=0) n = 0 lenline = len(r_node.body) while n < lenline and not isinstance(r_node.body[n], ast.ImportFrom) and not isinstance( r_node.body[n], ast.Import): n += 1 while n < lenline and (isinstance(r_node.body[n], ast.ImportFrom)): n += 1 r_node.body.insert(n, npu_import) log_msg(n, "from npu_bridge.estimator.npu import npu_convert_dropout")
def add_npu_func_to_params(node, param_index, org_func_name, param_name, npu_func, npu_func_args): """Add npu function to parameters""" param_node = None if ((not util_global.get_value("distributed_mode", "") or util_global.get_value("distributed_mode", "") == "horovod") and (param_name in ("callbacks", "hooks", "optimizer"))): return node log_param_msg = "".join([org_func_name, " add npu ", param_name]) log_msg(getattr(node, "lineno", "None"), log_param_msg) for index, _ in enumerate(node.args): if param_index is not None and index == param_index: param_node = node.args.pop(param_index) for keyword in node.keywords: if keyword.arg == param_name: param_node = keyword if param_node: if isinstance(param_node, ast.keyword): new_value = ast.Call(func=ast.Name(id=npu_func, ctx=ast.Load()), args=[], keywords=[ ast.keyword(arg=npu_func_args, value=param_node.value) ]) ast.copy_location(new_value, param_node.value) param_node.value = new_value else: node.keywords.append( ast.keyword(arg=param_name, value=ast.Call(func=ast.Name(id=npu_func, ctx=ast.Load()), args=[], keywords=[ ast.keyword(arg=npu_func_args, value=param_node) ]))) else: node.keywords.append( ast.keyword(arg=param_name, value=pasta.parse("".join([npu_func, "()"])))) return node
def insert_into_db(mps): columns = ['id', 'title', 'source', 'a_type', 'username', 'description', 'content', 'source_url', 'publish_time', 'category', 'tags', 'cover', 'images', 'views', 'comments'] keys = ['id', 'title', 'from', ('isOriginal',1), 'source', 'abstract', 'content', 'url', 'publishtime', 'category', 'tags', 'cover', 'images', ('views',1), ('comments',1)] tt = [("'{%s}'" % o) if isinstance(o,str) else ("{%s}" % o[0]) for o in keys] table = "tb_article_crawl" c2k_mp = {o[0]:o[1] for o in zip(columns,tt)} update_columns = ["views","comments"] data = [] for mp in mps: mp["isOriginal"] = 1 if mp["isOriginal"] else 2 mp["from"] = "今日头条" np = {k:(re.sub("[\r\n\t ]+"," ",mp[k]).replace("'","\\'") if isinstance(mp[k],str) else mp[k]) for k in mp.keys()} temp_temp = "(%s) %s" % (",".join(tt), "on duplicate key update "+",".join(["`%s`=%s" % (o,c2k_mp[o]) for o in update_columns]) if len(update_columns)>0 else "") try: data.append(temp_temp.format_map(np)) except Exception: tp, e,trace = sys.exc_info() util.log_msg("format data error:%s" % str(e)) db_dump.dump_db(table,columns,data)
def replace_image(content,referer): tree = etree.HTML(content) nodes = tree.xpath("//img") imgs = [] for node in nodes: keys = node.keys() key = "src" if "src" in keys else ("href" if "href" in keys else None) if key is not None: src = util.parse_url(referer,node.get(key)) if src.find("?") > -1: # not good here util.log_msg("expecting image url with no params%s" % src) continue _,name,nsrc = imge_transfer(src,referer) node.set(key,nsrc) imgs.append((src,name)) target = "".join([etree.tounicode(node) for node in tree.xpath("/html/body/*")]) imgs_str = "|".join([o[1] for o in imgs]) return target,imgs_str
def convert_hvd_distributed_api(node): """Convert horovod distributed APIs""" log_msg( getattr(node, "lineno", "None"), 'change hvd.DistributedOptimizer to npu_distributed_optimizer_wrapper') node.func = ast.Name(id="npu_distributed_optimizer_wrapper", ctx=ast.Load()) opt_keyword = None for keyword in node.keywords: if keyword.arg == "optimizer": opt_keyword = keyword node.keywords.clear() if opt_keyword is None: opt_arg = node.args[0] node.args.clear() node.args.append(opt_arg) else: node.keywords.append(opt_keyword) util_global.set_value('need_conver', True) return node
def load_cluster_info(G=None, clusters_dir="/notebooks/nx_graph_utils/hemibrain/und_flybrain_v1.1/"): """Loads cluster info into the provided graph. If no graph is specified, loads the flybrain data with `get_FlyBrain_graph`. Returns the graph as well as two dictionaries with information about the clusters. """ n_clusters = {} cluster_hist = {} if G is None: G = get_FlyBrain_graph() for filename in os.listdir(clusters_dir): log_msg("Reading file", filename) if "unweighted" in filename: cluster_name = "unweighted" elif "lr" in filename: cluster_name = "lr" else: try: cluster_name = (filename.split(".")[0]).split("_")[2] cluster_name = cluster_name[1:].replace("p", ".") except: log_msg("Error reading file", filename, ", skipping file") continue lines_read, clust_histogram = read_cluster_ids(G, filename, cluster_name) n_clusters[cluster_name] = clust_histogram.shape[0] cluster_hist[cluster_name] = clust_histogram log_msg(n_clusters) return G, n_clusters, cluster_hist
def insert_npu_import(r_node): """Add NPU import modules""" npu_alias = ast.alias(name='*', asname=None) npu_import = ast.ImportFrom(module='npu_bridge.npu_init', names=[npu_alias], level=0) num = 5 if len(r_node.body) >= 5 else len(r_node.body) import_index = 0 is_insert = False for i in range(0, num): if isinstance(r_node.body[i], ast.Import): r_node.body.insert(i, npu_import) log_msg(i, "from npu_bridge.npu_init import *") is_insert = True break if isinstance(r_node.body[i], ast.ImportFrom): if r_node.body[i].module != "__future__": r_node.body.insert(i, npu_import) log_msg(i, "from npu_bridge.npu_init import *") is_insert = True break import_index = i + 1 if not is_insert: r_node.body.insert(import_index, npu_import) log_msg(import_index, "from npu_bridge.npu_init import *")
def get_article(url,referer,data,fp,result2): url, h = util.get_page({"url":url,"headers":{"Referer":referer}}) tree = etree.HTML(h) scripts = [o for o in tree.xpath("//script/text()") if o.find("BASE_DATA") > -1 or o.find("__pgcInfo")>-1] scripts.append("console.log(JSON.stringify(BASE_DATA))") open("data/tmp.js","w").write("\n".join(scripts)) r = get_params("data/tmp.js") if r is not None: mp = json.loads(r) obj = {"entry":data,"data":mp} conf = [("title",["data","artilceInfo","title"]), ("content",["data","artilceInfo","content"],None,html.unescape), ("comments",["data","commentInfo","comments_count"],0), ("isOriginal",["data","artilceInfo","subInfo","isOriginal"],False), ("url",["__const",url]), ("views",["entry","go_detail_count"], 0), ("cover",["entry","image_url"],""), ("abstract",["entry","abstract"], ""), ("source",["data","artilceInfo","subInfo","source"],""), ("publishtime",["data","artilceInfo","subInfo","time"]), ("tags",["data","artilceInfo","tagInfo","tags"],"",lambda o:",".join([so["name"] for so in o])), ("category",["data","headerInfo","chineseTag"],""), ] result = {} for cf in conf: v = util.get_jpath(obj,cf[1],cf[2] if len(cf)>2 else None,cf[3] if len(cf)>3 else None) if v is not None: result[cf[0]] = v result["id"] = hashlib.md5(url.encode("utf-8")).hexdigest() if "content" in result: result["content"],result["images"] = replace_image(result["content"],url) if "cover" in result and len(result["cover"])>0: result["cover"] = imge_transfer(util.parse_url(url,result["cover"]),url)[1] if len(result) > 0: result2.append(result) else: util.log_msg("could not parse content from html file,need to check this out.url:%s,referer:%s." % (url,referer)) else: util.log_msg("could not parse data from html file,need to check this out.url:%s,referer:%s." % (url,referer))
def convert_distributed_strategy_apis(node): """Convert distributed strategy API""" if isinstance(node.func, ast.Attribute) and isinstance( node.func.value, ast.Attribute): if ("Optimizer" in node.func.attr and node.func.attr != "ScipyOptimizerInterface" and node.func.attr != "MixedPrecisionLossScaleOptimizer"): log_msg(getattr(node, "lineno", "None"), "add npu distribute optimizer to tensorflow optimizer") new_node = ast.Call(func=ast.Name( id="npu_distributed_optimizer_wrapper", ctx=ast.Load()), args=[node], keywords=[]) ast.copy_location(new_node, node) util_global.set_value('need_conver', True) return new_node if isinstance( node.func, ast.Name ) and "Optimizer" in node.func.id and node.func.id != "NPULossScaleOptimizer": log_msg(getattr(node, "lineno", "None"), "add npu distribute optimizer to tensorflow optimizer") new_node = ast.Call(func=ast.Name( id="npu_distributed_optimizer_wrapper", ctx=ast.Load()), args=[node], keywords=[]) ast.copy_location(new_node, node) util_global.set_value('need_conver', True) return new_node if _call_name_match(node.func, "TrainSpec"): return convert_origin_func_to_npu( node, tf_func_map["tf.estimator.TrainSpec"], "TrainSpec", ["hooks"]) if _call_name_match(node.func, "EvalSpec"): return convert_origin_func_to_npu(node, tf_func_map["tf.estimator.EvalSpec"], "EvalSpec", ["hooks"]) if isinstance(node.func, ast.Attribute) and node.func.attr == "train": if isinstance(node.func.value, ast.Attribute) and node.func.value.attr == "learning": return node return convert_origin_func_to_npu( node, tf_func_map["tf.estimator.Estimator.train"], "Estimator.train", ["hooks"], True) if isinstance(node.func, ast.Attribute) and (node.func.attr == 'compile'): if isinstance(node.func.value, ast.Name) and node.func.value.id == "re": return node return convert_origin_func_to_npu( node, tf_func_map["tf.keras.Model.compile"], "Model.compile", ["optimizer"], True) if isinstance(node.func, ast.Attribute) and node.func.attr == "fit": return convert_origin_func_to_npu(node, tf_func_map["tf.keras.Model.fit"], "Model.fit", ["callbacks"], True) if isinstance(node.func, ast.Attribute) and node.func.attr == "fit_generator": return convert_origin_func_to_npu( node, tf_func_map["tf.keras.Model.fit_generator"], "Model.fit_generator", ["callbacks"], True) if isinstance(node.func, ast.Attribute) and node.func.attr == "gradients" and \ isinstance(node.func.value, ast.Name) and node.func.value.id == "tf": return convert_tf_gradient_distributed(node) return node
"""Testing the functionality of the ergm package""" import sys import time import numpy as np import networkx as nx from ergm import ERGM from util import log_msg from scipy.special import binom log_msg("BEGIN SCRIPT:", __file__) log_msg("Testing ergms with adjacency matrices") p0 = 0.1 # edge density for ER graph n_nodes = 6 n_samples = 10000 seed = 17289 # Not actually used at the moment np.random.seed(seed) m = int(binom(n_nodes, 2)) log_msg("Using networkx to sample Erdos-Renyi random graphs with edge probability p = {}".format(p0)) log_msg("Producing {} samples with {} nodes".format(n_samples, n_nodes)) log_msg("Using seed {} for numpy; using 17 * k for nx random graphs".format(seed)) nx_ER_start = time.time() nx_ER_list = [nx.gnp_random_graph(n_nodes, p0, seed=17 * k) for k in range(n_samples)] nx_ER_end = time.time()
def _MLE_sampler(self, observed, n_estim_samples=100, alpha=1, alpha_rate=0.999, max_iter=1000, x_tol=1e-8, print_logs=None, sampler_logs=None, **kwargs): """ Compute the maximum likelihood estimate (MLE) of parameters for the observed data using gradient ascent on the likelihood. The expected value of the current ERGM is estimated by sampling. :param observed: Observed graphs, shape n_nodes x n_nodes x n_samples :param n_estim_samples: samples to use at each estimation step :param alpha: Scale factor of gradient. :param max_iter: Maximum number of iterations to take :param x_tol: Quit trying to optimize if gradient norm falls below x_tol :param print_logs: File pointer, where to print logs (default None to suppress output) :param sampler_logs: File pointer, where sampler method should print logs (default None to suppress output) :param kwargs: Additional keyword arguments are passed to sampler :return: Dictionary including trajectory and such. """ if len(observed.shape) == 2: k_obs = self.stats(observed) log_msg("MLE_sampler: Passed single graph; observed stats:\n", k_obs, out=print_logs) else: try: all_obs_k = np.array([ self.stats(observed[:, :, i]) for i in range(observed.shape[2]) ]) except: # if sparse arrays were used, observed will be an array of sparse matrices all_obs_k = np.array([ self.stats(observed[i].tocsr()) for i in range(observed.shape[0]) ]) k_obs = all_obs_k.mean(axis=0) log_msg("MLE_sampler: Computed stats, resulting shape:", all_obs_k.shape, out=print_logs) log_msg("MLE_sampler: average stats:", k_obs, out=print_logs) log_msg("MLE_sampler: %8d estimate samples" % n_estim_samples, out=print_logs) log_msg("MLE_sampler: %8f alpha" % alpha, out=print_logs) log_msg("MLE_sampler: %8d max iterations" % max_iter, out=print_logs) log_msg("MLE_sampler: %8e L tolerance" % x_tol, out=print_logs) # trajectory = np.zeros((max_iter, self.theta.shape[0])) n_nodes = observed.shape[0] # trajectory of thetas and such theta_t = np.zeros((max_iter + 1, *self.theta.shape)) k_bar_t = np.zeros_like(theta_t) grad_t = np.zeros_like(theta_t) covar_t = np.zeros( (max_iter + 1, *self.theta.shape, *self.theta.shape)) # local variables of the algorithm delta_theta = np.zeros_like(self.theta) theta_t[0, :] = self.theta # store initial value of theta # the sample values at each iteration # The actual sample graphs are not needed, so for now, I'll just ignore them. If they turn out to be useful, # we can sort out how to handle sparse vs. non-sparse ERGMs. # if self.use_sparse: # G_samp = np.array([sparse.lil_matrix(0) for _ in range(n_estim_samples)]) # else: # G_samp = np.zeros((n_nodes, n_nodes, n_estim_samples), dtype=int) k_samp = np.zeros((*self.theta.shape, n_estim_samples)) log_msg( f"{'iter':4} {'|theta(t)|':20} {'|gradient|':20} {'alpha':20} {'|Delta theta|':20}", out=print_logs) stopping_criteria = [] stop_iter = -1 for step in range(max_iter): try: _, k_samp[:, :] = self.sample_gibbs(n_nodes, n_estim_samples, print_logs=sampler_logs, **kwargs) k_bar_t[step, :] = k_samp.mean(axis=1) covar_t[step, :, :] = np.cov(k_samp) grad_t[step, :] = k_obs - k_bar_t[step, :] grad_norm = np.linalg.norm(grad_t[step, :]) delta_theta[:] = alpha * grad_t[step, :] log_msg( f"{step:4d} {np.linalg.norm(theta_t[step, :]):20.8f} {grad_norm:20.8f} {alpha:20.8f} {alpha * grad_norm:20.8f}", out=print_logs) theta_t[step + 1, :] = theta_t[step, :] + delta_theta self._set_theta(theta_t[step + 1, :], True) # update alpha alpha = alpha * alpha_rate # check stopping criteria # TODO implement 2nd order stopping criteria if step + 1 == max_iter: stopping_criteria.append("max_iter reached") if grad_norm < x_tol: stopping_criteria.append("grad_norm < x_tol") if len(stopping_criteria) > 0: stop_iter = step # break except KeyboardInterrupt: stopping_criteria.append("keyboard interrupt") stop_iter = step - 1 # break except Exception as e: stopping_criteria.append("unhandled exception: {}".format(e)) stop_iter = step - 1 # break finally: if len(stopping_criteria) > 0: if stop_iter < 0: stop_iter = step break trajectory = { "theta": theta_t[:stop_iter, :], "expected stats": k_bar_t[:stop_iter, :], "covariances": covar_t[:stop_iter, :, :], "gradient": grad_t[:stop_iter, :], "stop": stopping_criteria, "stop_iter": stop_iter } return trajectory
def check_posts(posts, post_type, forced): """ main function for checking submissions and replies made in subreddit; checks for iconified monster links and 'flair in ID' messages """ for post in posts: called = False # skip if post made by bot if str(post.author) == USERNAME: continue # check processed posts and ignore lists if forced is False: if post_type == 'SUBMISSIONS': if str(post.id) in processed_submissions: continue if str(post.id) in ignored_submissions: continue elif post_type == 'COMMENTS': if str(post.id) in processed_comments: continue if str(post.link_id)[3:] in ignored_submissions: continue if post_type == 'SUBMISSIONS': temp_text = post.selftext.encode('utf-8') elif post_type == 'COMMENTS': temp_text = post.body.encode('utf-8') # check callouts if temp_text.find('-/u/tamabot') > -1: log_msg('Found negative callout') pass elif temp_text.find('/u/tamabot') > -1: log_msg('Found callout request!') called = True # check for Monster Icons - disabled # if called is True or forced is True: # n, temp_id, msg, listed = 0, 0, [], [] # for m in pattern_icon.finditer(temp_text): # e = m.groupdict() # if e['id'] is not None: # temp_id = int(e['id']) # elif e['sid'] is not None and e['cid'] is not None: # temp_id = (int(e['sid'][1:]) * 50) + int(e['cid'][1:]) + 1 # n, listed, msg = process_monsters(temp_id, n, listed, msg) # # check for PADX Team # m = re.findall(pattern_padxsim, temp_text, re.I | re.U) # if m: # for e in m: # for i in e: # temp_id = int(i) # n, listed, msg = process_monsters(temp_id, n, listed, msg) # # create Monster Table # if n > 0: # create_post(post, msg, post_type, 'MONSTER TABLE') # check for Flair Call m = re.search(pattern_flair_call, temp_text, re.I | re.U) if m or called is True or forced is True: msg = [] if post.author_flair_text is not None: msg.append("Found %s's flair: **%s**\n" % (str(post.author), post.author_flair_text)) create_post(post, msg, post_type, 'FLAIR ID') # update processed posts list if post_type == 'SUBMISSIONS': check_processed_posts(processed_submissions, str(post.id), LIMIT) elif post_type == 'COMMENTS': check_processed_posts(processed_comments, str(post.id), LIMIT) sleep(SLEEP_LONG)
def check_pm(msgs): """ function for checking bot's private messages; delete if parent commentor (or subreddit moderator) has requested post deletion; stops bot if moderator has requested a halt """ for msg in msgs: # check for delete request m = re.search(ur'^\+delete\s(.+?)$', msg.body.lower()) if m: id = "t1_%s" % m.group(1) c = r.get_info(thing_id = id) if c is not None: c_parent = r.get_info(thing_id = c.parent_id) if c_parent.author is None: delete_post(c, 'PM', '', msg.author.name) else: if msg.author.name == c_parent.author.name or msg.author.name in mod_list or msg.author.name == 'mrmin123': if "Please request this post to be deleted to un-ignore." in c.body: log_msg("Un-ignoring posts under %s by %s's request" % (c_parent.permalink, msg.author.name)) update_db(log_coll, stat_coll, 'UNIGNORE', c_parent.permalink, msg.author.name) try: ignored_submissions.remove(str(c.parent_id)[3:]) except Exception as e: log_error(e) delete_post(c, 'PM', c_parent.permalink, msg.author.name) else: log_warning("Incorrect delete request from %s for %s" % (msg.author.name, c.permalink)) update_db(log_coll, stat_coll, 'DEL_BAD', c.permalink, msg.author.name) # check for ignore request m = re.search(ur'^\+ignore\s(.+?)$', msg.body.lower()) if m: id = "t3_%s" % m.group(1) c = r.get_info(thing_id = id) if c.author is not None and (msg.author.name == c.author.name or msg.author.name in mod_list or msg.author.name == 'mrmin123'): if m.group(1) not in ignored_submissions: ignored_submissions.append(m.group(1)) log_msg("Ignoring posts under %s by %s's request" % (c.short_link, msg.author.name)) update_db(log_coll, stat_coll, 'IGNORE_PM', c.short_link, msg.author.name) temp_msg = "%s\nI am ignoring any new posts in this thread by OP/moderator's request! Please request this post to be deleted to un-ignore.\n" % signature_intro create_post(c, [temp_msg], 'SUBMISSIONS', 'IGNORE_POST') else: if type(c) is praw.objects.Submission: tempLink = c.short_link elif type(c) is praw.objects.Comment: tempLink = c.permalink else: tempLink = m.group(1) log_warning("Incorrect ignore request from %s for %s" % (msg.author.name, tempLink)) update_db(log_coll, stat_coll, 'IGNORE_BAD', tempLink, msg.author.name) # check for revisit m = re.search(ur'^\+visit\s(.+?)$', msg.body.lower()) if m: temp_type = 'SUBMISSIONS' id = "t3_%s" % m.group(1) c = r.get_info(thing_id = id) if c is None: temp_type = 'COMMENTS' id = id = "t1_%s" % m.group(1) c = r.get_info(thing_id = id) if c is not None and c.subreddit.display_name.lower() == SUBREDDIT.lower() and (msg.author.name == c.author.name or msg.author.name in mod_list or msg.author.name == 'mrmin123'): log_msg("Revisiting %s under %s's request" % (c.permalink, msg.author.name)) update_db(log_coll, stat_coll, 'REVISIT', c.permalink, msg.author.name) check_posts([c], temp_type, True) else: if type(c) is praw.objects.Submission: tempLink = c.short_link elif type(c) is praw.objects.Comment: tempLink = c.permalink else: tempLink = m.group(1) log_msg("Incorrect revisit request for %s by %s" % (tempLink, msg.author.name)) update_db(log_coll, stat_coll, 'REVISIT_BAD', tempLink, msg.author.name) # check for moderator halt request if msg.author.name in mod_list or msg.author.name == 'mrmin123': m = re.search(ur'^\+halt$', msg.body.lower()) if m: msg.mark_as_read() log_error("Bot halt requested by %s" % msg.author.name) update_db(log_coll, stat_coll, 'HALT', '', msg.author.name) exit() msg.mark_as_read() sleep(SLEEP) sleep(SLEEP_LONG)
def convert_loss_scale_api(node): if isinstance(node.func, ast.Attribute): if node.func.attr == "FixedLossScale": log_msg( getattr(node, 'lineno', 'None'), "change tf.train.experimental.FixedLossScale" " to FixedLossScaleManager") node.func = ast.Name(id="FixedLossScaleManager", ctx=ast.Load()) if len(node.keywords) == 1: node.keywords[0].arg = "loss_scale" util_global.set_value('need_conver', True) return node if node.func.attr == "DynamicLossScale": log_msg( getattr(node, 'lineno', 'None'), "change tf.train.experimental.DynamicLossScale" " to ExponentialUpdateLossScaleManager") node.func = ast.Name(id="ExponentialUpdateLossScaleManager", ctx=ast.Load()) initial_loss_scale = None increment_period = None multiplier = None for index, arg in enumerate(node.args): if index == 0: initial_loss_scale = arg if index == 1: increment_period = arg if index == 2: multiplier = arg for keyword in node.keywords: if keyword.arg == "initial_loss_scale": keyword.arg = "init_loss_scale" initial_loss_scale = keyword if keyword.arg == "increment_period": keyword.arg = "incr_every_n_steps" increment_period = keyword if keyword.arg == "multiplier": keyword.arg = "incr_ratio" multiplier = keyword if initial_loss_scale: if not isinstance(initial_loss_scale, ast.keyword): node.keywords.append( ast.keyword(arg="init_loss_scale", value=initial_loss_scale)) else: node.keywords.append( ast.keyword(arg="init_loss_scale", value=pasta.parse("2**15"))) if increment_period: if not isinstance(increment_period, ast.keyword): node.keywords.append( ast.keyword(arg="incr_every_n_steps", value=increment_period)) else: node.keywords.append( ast.keyword(arg="incr_every_n_steps", value=pasta.parse("2000"))) if multiplier: if not isinstance(multiplier, ast.keyword): node.keywords.append( ast.keyword(arg="incr_ratio", value=multiplier)) else: node.keywords.append( ast.keyword(arg="incr_ratio", value=pasta.parse("2"))) node.args = [] util_global.set_value('need_conver', True) return node if node.func.attr == "MixedPrecisionLossScaleOptimizer": log_msg( getattr(node, 'lineno', 'None'), "change tf.train.experimental.MixedPrecisionLossScaleOptimizer" " to NPULossScaleOptimizer") node.func = ast.Name(id="NPULossScaleOptimizer", ctx=ast.Load()) for keyword in node.keywords: if keyword.arg == "loss_scale": keyword.arg = "loss_scale_manager" util_global.set_value('need_conver', True) return node
def ast_call(node): convert_loss_scale_api(node) if _call_name_match(node.func, "set_experimental_options"): log_msg( getattr(node, 'lineno', 'None'), 'change set_experimental_options(*) to set_experimental_options(experimental_options)' ) node.args = [ast.Name(id='experimental_options', ctx=ast.Load())] node.keywords = [] util_global.set_value('need_conver', True) if isinstance(node.func, ast.Name) and node.func.id == 'check_available_gpus': log_msg(getattr(node, 'lineno', 'None'), "change check_available_gpus() to ['/device:CPU:0']") util_global.set_value('need_conver', True) return ast.List(elts=[ast.Str(s="/device:CPU:0")], ctx=ast.Load()) if (isinstance(node.func, ast.Name) and node.func.id == 'ConfigProto') or \ (isinstance(node.func, ast.Attribute) and node.func.attr == 'ConfigProto'): log_success_report(getattr(node, 'lineno', 'None'), 'ConfigProto()') src = copy.deepcopy(node) node.func = ast.Name(id='npu_config_proto', ctx=ast.Load()) node.args = [] node.keywords = [] node.keywords.append(ast.keyword(arg='config_proto', value=src)) util_global.set_value('need_conver', True) return node if (isinstance(node.func, ast.Name) and node.func.id == 'GraphOptions') or \ (isinstance(node.func, ast.Attribute) and node.func.attr == 'GraphOptions'): log_success_report(getattr(node, 'lineno', 'None'), 'GraphOptions()') src = copy.deepcopy(node) node.func = ast.Name(id='npu_graph_options', ctx=ast.Load()) node.args = [] node.keywords = [] node.keywords.append(ast.keyword(arg='graph_options', value=src)) util_global.set_value('need_conver', True) return node if (isinstance(node.func, ast.Name) and node.func.id == 'OptimizerOptions') or \ (isinstance(node.func, ast.Attribute) and node.func.attr == 'OptimizerOptions'): log_success_report(getattr(node, 'lineno', 'None'), 'OptimizerOptions()') src = copy.deepcopy(node) node.func = ast.Name(id='npu_optimizer_options', ctx=ast.Load()) node.args = [] node.keywords = [] node.keywords.append(ast.keyword(arg='optimizer_options', value=src)) util_global.set_value('need_conver', True) return node if (isinstance(node.func, ast.Name) and node.func.id == 'Session') or \ (isinstance(node.func, ast.Attribute) and node.func.attr == 'Session'): log_success_report(getattr(node, 'lineno', 'None'), 'Session()') config = None for index, _ in enumerate(node.args): if index == 2: config = node.args.pop(2) break for keyword in node.keywords: if keyword.arg == 'config': config = keyword break if config: if isinstance(config, ast.keyword): config.value = ast.Call(func=ast.Name(id='npu_config_proto', ctx=ast.Load()), args=[], keywords=[ ast.keyword(arg='config_proto', value=config.value) ]) else: node.keywords.append( ast.keyword(arg='config', value=ast.Call( func=ast.Name(id='npu_config_proto', ctx=ast.Load()), args=[], keywords=[ ast.keyword(arg='config_proto', value=config) ]))) else: node.keywords.append( ast.keyword(arg='config', value=ast.Call(func=ast.Name(id='npu_config_proto', ctx=ast.Load()), args=[], keywords=[]))) util_global.set_value('need_conver', True) return node if isinstance(node.func, ast.Attribute ) and node.func.attr == "BroadcastGlobalVariablesHook": log_success_report(getattr(node, "lineno", "None"), 'BroadcastGlobalVariablesHook') node.func = ast.Name(id="NpuEmptyHook", ctx=ast.Load()) node.args = [] node.keywords = [] util_global.set_value('need_conver', True) return node if isinstance(node.func, ast.Attribute) and node.func.attr == "DistributedOptimizer": log_success_report(getattr(node, "lineno", "None"), 'DistributedOptimizer') return node.args[0] if isinstance(node.func, ast.Attribute) and node.func.attr == 'shard': log_success_report(getattr(node, "lineno", "None"), 'shard') node.args = [ ast.Call(func=ast.Name(id='get_rank_size', ctx=ast.Load()), args=[], keywords=[]), ast.Call(func=ast.Name(id='get_rank_id', ctx=ast.Load()), args=[], keywords=[]) ] util_global.set_value("has_hccl_api", True) util_global.set_value('need_conver', True) if isinstance(node.func, ast.Attribute) and node.func.attr == 'dropout': if isinstance(node.func.value, ast.Attribute) and node.func.value.attr == 'nn': log_success_report(getattr(node, "lineno", "None"), 'dropout') node.func = ast.Attribute(value=ast.Name(id='npu_ops', ctx=ast.Load()), attr='dropout', ctx=ast.Load()) keywords_new = [] for keyword in node.keywords: if keyword.arg != 'rate': keywords_new.append(keyword) node.keywords = keywords_new util_global.set_value('need_conver', True) if isinstance(node.func, ast.Attribute) and ((node.func.attr == 'map_and_batch') or (node.func.attr == 'batch' \ and (not isinstance(node.func.value, ast.Attribute) or (isinstance(node.func.value, ast.Attribute) and node.func.value.attr != 'train')))): exist = False for keyword in node.keywords: if keyword.arg == 'drop_remainder': exist = True if ((isinstance(keyword.value, ast.NameConstant) and keyword.value.value != True) or (not isinstance(keyword.value, ast.NameConstant))): log_success_report(getattr(node, "lineno", "None"), node.func.attr) keyword.value = pasta.parse('True') util_global.set_value('need_conver', True) if not exist: log_success_report(getattr(node, "lineno", "None"), node.func.attr) keyword = ast.keyword(arg='drop_remainder', value=pasta.parse('True')) node.keywords.insert(0, keyword) util_global.set_value('need_conver', True) if (isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name) and node.func.value.id == 'tf' and node.func.attr == 'device'): log_success_report(getattr(node, "lineno", "None"), node.func.attr) node.args = [ast.Str(s='/cpu:0')] util_global.set_value('need_conver', True) if isinstance(node.func, ast.Attribute) and ( node.func.attr == "get_distribution_strategy" or node.func.attr == "MirroredStrategy" or node.func.attr == "MultiWorkerMirroredStrategy"): log_success_report(getattr(node, "lineno", "None"), node.func.attr) new_func = ast.Attribute(value=ast.Name(id="npu_strategy", ctx=ast.Load()), attr="NPUStrategy", ctx=ast.Load()) ast.copy_location(new_func, node.func) node.func = new_func node.keywords = [] node.args = [] util_global.set_value('need_conver', True) if isinstance(node.func, ast.Attribute) and (node.func.attr == 'AdviceProto'): log_success_report(getattr(node, 'lineno', 'None'), 'tf.profiler.AdviceProto') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'Checker'): log_success_report(getattr(node, 'lineno', 'None'), 'tf.profiler.AdviceProto.Checker') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'CheckersEntry'): log_success_report(getattr(node, 'lineno', 'None'), 'tf.profiler.AdviceProto.CheckersEntry') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'GraphNodeProto'): log_success_report(getattr(node, 'lineno', 'None'), 'tf.profiler.GraphNodeProto') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'InputShapesEntry'): log_success_report(getattr(node, 'lineno', 'None'), 'tf.profiler.GraphNodeProto.InputShapesEntry') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'MultiGraphNodeProto'): log_success_report(getattr(node, 'lineno', 'None'), 'tf.profiler.MultiGraphNodeProto') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'OpLogProto'): log_success_report(getattr(node, 'lineno', 'None'), 'tf.profiler.OpLogProto') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'IdToStringEntry'): log_success_report(getattr(node, 'lineno', 'None'), 'tf.profiler.OpLogProto.IdToStringEntry') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'ProfileOptionBuilder'): log_success_report(getattr(node, 'lineno', 'None'), 'tf.profiler.ProfileOptionBuilder') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'advise'): log_success_report(getattr(node, 'lineno', 'None'), 'tf.profiler.advise') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'profile'): log_success_report(getattr(node, 'lineno', 'None'), 'tf.profiler.profile') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'write_op_log'): log_success_report(getattr(node, 'lineno', 'None'), 'tf.profiler.write_op_log') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'TPUEstimator') and \ ((isinstance(node.func.value, ast.Attribute) and (node.func.value.attr == 'tpu')) or \ (isinstance(node.func.value, ast.Name) and (node.func.value.id == 'tpu'))): add_eval_on_tpu = True add_use_tpu = True add_export_to_tpu = True for keyword in node.keywords: if (keyword.arg == 'eval_on_tpu') or ( keyword.arg == 'use_tpu') or (keyword.arg == 'export_to_tpu'): if (not isinstance(keyword.value, ast.NameConstant)) or ( isinstance(keyword.value, ast.NameConstant) and (keyword.value.value != False)): log_success_report(getattr(node, 'lineno', 'None'), 'TPUEstimator(' + keyword.arg + '=*)') keyword.value = pasta.parse('False') util_global.set_value('need_conver', True) if add_eval_on_tpu and (keyword.arg == 'eval_on_tpu'): add_eval_on_tpu = False if add_use_tpu and (keyword.arg == 'use_tpu'): add_use_tpu = False if add_export_to_tpu and (keyword.arg == 'export_to_tpu'): add_export_to_tpu = False if add_eval_on_tpu: log_success_report(getattr(node, 'lineno', 'None'), 'TPUEstimator(eval_on_tpu=*)') node.keywords.append( ast.keyword(arg='eval_on_tpu', value=pasta.parse('False'))) util_global.set_value('need_conver', True) if add_use_tpu: log_success_report(getattr(node, 'lineno', 'None'), 'TPUEstimator(use_tpu=*)') node.keywords.append( ast.keyword(arg='use_tpu', value=pasta.parse('False'))) util_global.set_value('need_conver', True) if add_export_to_tpu: log_success_report(getattr(node, 'lineno', 'None'), 'TPUEstimator(export_to_tpu=*)') node.keywords.append( ast.keyword(arg='export_to_tpu', value=pasta.parse('False'))) util_global.set_value('need_conver', True) if isinstance(node.func, ast.Attribute) and (node.func.attr == 'VirtualDeviceConfiguration'): log_success_report(getattr(node, 'lineno', 'None'), 'VirtualDeviceConfiguration') util_global.set_value('need_conver', True) memory_limit = None for keyword in node.keywords: if keyword.arg == 'memory_limit': memory_limit = keyword break if memory_limit: memory_limit.value = ast.NameConstant(value=None) else: node.keywords.append( ast.keyword(arg='memory_limit', value=ast.NameConstant(value=None))) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'set_soft_device_placement'): log_success_report(getattr(node, 'lineno', 'None'), 'set_soft_device_placement') util_global.set_value('need_conver', True) node.args = [] node.keywords = [ ast.keyword(arg='enabled', value=ast.NameConstant(value=True)) ] return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'set_memory_growth'): log_success_report(getattr(node, 'lineno', 'None'), 'set_memory_growth') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'set_virtual_device_configuration'): log_success_report(getattr(node, 'lineno', 'None'), 'set_virtual_device_configuration') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'jit_scope'): if isinstance(node.func.value, ast.Attribute) and (node.func.value.attr == 'experimental'): if isinstance(node.func.value.value, ast.Attribute) and (node.func.value.value.attr == 'xla'): log_success_report(getattr(node, 'lineno', 'None'), '*.xla.experimental.jit_scope') util_global.set_value('need_conver', True) compile_ops = None for keyword in node.keywords: if keyword.arg == 'compile_ops': compile_ops = keyword break if compile_ops: compile_ops.value = pasta.parse('False') else: node.keywords.append( ast.keyword(arg='compile_ops', value=pasta.parse('False'))) return node for estimator in util_global.get_value('Estimators', []): if (isinstance(node.func, ast.Attribute) and (node.func.attr == estimator)) \ or (isinstance(node.func, ast.Name) and (node.func.id == estimator)): log_msg(getattr(node, 'lineno'), estimator + '() add config=npu_run_config_init()') config = None for keyword in node.keywords: if keyword.arg == 'config': config = keyword break if config: new_value = ast.Call(func=ast.Name(id='npu_run_config_init', ctx=ast.Load()), args=[], keywords=[ ast.keyword(arg='run_config', value=config.value) ]) ast.copy_location(new_value, config.value) config.value = new_value else: node.keywords.append( ast.keyword(arg='config', value=pasta.parse('npu_run_config_init()'))) util_global.set_value('need_conver', True) return node for estimator_func in util_global.get_value('EstimatorFunc', []): if isinstance(node.func, ast.Attribute) and (node.func.attr == estimator_func): if isinstance( node.func.value, ast.Attribute) and node.func.value.attr == "learning": return node input_fn = None hooks = None for index, _ in enumerate(node.args): if index == 0: input_fn = node.args[0] elif index == 1: hooks = node.args.pop(1) for keyword in node.keywords: if keyword.arg == 'input_fn': input_fn = keyword elif keyword.arg == 'hooks': hooks = keyword if not input_fn: break if not hooks: node.keywords.append( ast.keyword(arg='hooks', value=pasta.parse('npu_hooks_append()'))) elif isinstance(hooks, ast.keyword): new_value = ast.Call(func=ast.Name(id='npu_hooks_append', ctx=ast.Load()), args=[], keywords=[ ast.keyword(arg='hooks_list', value=hooks.value) ]) ast.copy_location(new_value, hooks.value) hooks.value = new_value else: node.keywords.append( ast.keyword(arg='hooks', value=ast.Call( func=ast.Name(id='npu_hooks_append', ctx=ast.Load()), args=[], keywords=[ ast.keyword(arg='hooks_list', value=hooks) ]))) util_global.set_value('need_conver', True) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'compile'): opt_map = { "adadelta": "tf.keras.optimizers.Adadelta()", "adagrad": "tf.keras.optimizers.Adagrad()", "adam": "tf.keras.optimizers.Adam()", "adamax": "tf.keras.optimizers.Adamax()", "ftrl": "tf.keras.optimizers.Ftrl()", "nadam": "tf.keras.optimizers.Nadam()", "rmsprop": "tf.keras.optimizers.RMSprop()", "sgd": "tf.keras.optimizers.SGD()" } for keyword in node.keywords: if keyword.arg == "optimizer": log_success_report(getattr(node, 'lineno', 'None'), 'KerasDistributeOptimizer') if isinstance(keyword.value, ast.Str): keras_opt = opt_map[keyword.value.s] npu_keras_opt = "npu_keras_optimizer(" + keras_opt + ")" keyword.value = pasta.parse(npu_keras_opt) util_global.set_value('need_conver', True) return node if isinstance(node.func, ast.Attribute) and isinstance( node.func.value, ast.Attribute): if (node.func.attr.find("Optimizer") != -1) and (node.func.attr != 'ScipyOptimizerInterface'): log_msg(getattr(node, "lineno", "None"), "add NPUDistributedOptimizer()") new_node = ast.Call(func=ast.Name(id="npu_tf_optimizer", ctx=ast.Load()), args=[node], keywords=[]) ast.copy_location(new_node, node) util_global.set_value('need_conver', True) return new_node if isinstance(node.func, ast.Attribute): opt_list = [ "Adadelta", "Adagrad", "Adam", "Adamax", "Ftrl", "Nadam", "RMSprop", "SGD" ] if node.func.attr in opt_list: log_success_report(getattr(node, "lineno", "None"), "KerasDistributeOptimizer") new_node = ast.Call(func=ast.Name(id="npu_keras_optimizer", ctx=ast.Load()), args=[node], keywords=[]) ast.copy_location(new_node, node) util_global.set_value('need_conver', True) return new_node if (isinstance(node.func, ast.Attribute) and (node.func.attr == 'MonitoredTrainingSession')) or \ (isinstance(node.func, ast.Name) and (node.func.id == 'MonitoredTrainingSession')): log_success_report(getattr(node, "lineno", "None"), 'MonitoredTrainingSession') hooks = None for index, _ in enumerate(node.args): if index == 4: hooks = node.args.pop(4) break for keyword in node.keywords: if keyword.arg == 'hooks': hooks = keyword break if not hooks: node.keywords.append( ast.keyword(arg='hooks', value=pasta.parse('npu_hooks_append()'))) elif isinstance(hooks, ast.keyword): new_value = ast.Call( func=ast.Name(id='npu_hooks_append', ctx=ast.Load()), args=[], keywords=[ast.keyword(arg='hooks_list', value=hooks.value)]) ast.copy_location(new_value, hooks.value) hooks.value = new_value else: node.keywords.append( ast.keyword(arg='hooks', value=ast.Call(func=ast.Name(id='npu_hooks_append', ctx=ast.Load()), args=[], keywords=[ ast.keyword(arg='hooks_list', value=hooks) ]))) util_global.set_value('need_conver', True) return node specs = {'TrainSpec': 2, 'EvalSpec': 3} for spec, hooks_index in specs.items(): if _call_name_match(node.func, spec): log_success_report(getattr(node, "lineno", "None"), spec) hooks = None for index, _ in enumerate(node.args): if index == hooks_index: hooks = node.args.pop(hooks_index) break for keyword in node.keywords: if keyword.arg == 'hooks': hooks = keyword break if not hooks: node.keywords.append( ast.keyword(arg='hooks', value=pasta.parse('npu_hooks_append()'))) elif isinstance(hooks, ast.keyword): new_value = ast.Call(func=ast.Name(id='npu_hooks_append', ctx=ast.Load()), args=[], keywords=[ ast.keyword(arg='hooks_list', value=hooks.value) ]) ast.copy_location(new_value, hooks.value) hooks.value = new_value else: node.keywords.append( ast.keyword(arg='hooks', value=ast.Call( func=ast.Name(id='npu_hooks_append', ctx=ast.Load()), args=[], keywords=[ ast.keyword(arg='hooks_list', value=hooks) ]))) util_global.set_value('need_conver', True) return node
def sample_gibbs(self, n_nodes, n_samples=1, burn_in=None, n_steps=None, g0=None, print_logs=None): """ Sample from this ensemble, returning a 3d numpy array which is `n_nodes x n_nodes x n_samples` and a 2d numpy array which is `d x n_samples`, where `d` is the number of statistics. The second array stores the statistics of each sample, to avoid recomputing them (e.g. in parameter estimation) :param n_nodes: Number of nodes in the graph :param n_samples: Number of samples to return :param burn_in: Number of burn-in steps :param n_steps: Number of steps between samples :param g0: Initial adjacency matrix to use. Default is previous internal state for sampler, if appropriate :param print_logs: where to print logs. Default is None, suppressing output :return: A numpy array of integers (the adjacency matrices) and a numpy array of floats (the statistics) This method uses Gibbs sampling. """ # TODO write up some details on the internals of this method in the docstring # Compute number of steps this MC will take if burn_in is None: if self.current_adj.shape[0] == n_nodes: burn_in = 0 else: burn_in = 3 * int(math.ceil( n_nodes * math.log(n_nodes))) * len(self.theta) # above is based on some rough estimates/simulations if n_steps is None: n_steps = int(math.ceil(math.log(n_nodes))) * len(self.theta) * 2 total_steps = burn_in + n_steps * n_samples # total steps taken by markov chain # initialize output arrays if self.use_sparse: samples = np.array([ sparse.csr_matrix((n_nodes, n_nodes), dtype=int) for _ in range(n_samples) ]) else: samples = np.zeros((n_nodes, n_nodes, n_samples), dtype=int) sample_stats = np.zeros((self.theta.shape[0], n_samples)) # determine the sequence of edges to visit urand = np.random.rand(total_steps) # rand_indexes = np.random.choice(range(n_nodes * (n_nodes - 1) // (1 + (not self.directed))), size=total_steps, replace=True) rand_indexes = np.random.randint(0, n_nodes * (n_nodes - 1) // (1 + (not self.directed)), size=total_steps) edge_sequence = index_to_edge(rand_indexes, n_nodes, self.directed) # TODO There might be a way to speed up indexing into a CSR matrix by reformatting the edge sequence if g0 is None and self.current_adj.shape[0] == n_nodes: log_msg("sample_gibbs: previous adjacency matrix found", out=print_logs) # burn_in = 0 # we're picking up where we left off # pass elif g0 is None: log_msg("sample_gibbs: using empty graph for initial state", out=print_logs) # self.current_adj = np.zeros((n_nodes, n_nodes)) # self.current_stats = self.stats(self.current_adj) # self.current_logweight = np.dot(self.current_stats, self.theta) # TODO enable random initialization if self.use_sparse: self._initialize_sparse_adj(n_nodes, edge_sequence, reset_stats=True) else: self._initialize_dense_adj(n_nodes, reset_stats=True) self.proposed_stats = np.zeros_like(self.current_stats) # self.current_logweight = self.logweight(self.current_adj) else: log_msg( "sample_gibbs: using provided adjacency matrix for initial state", out=print_logs) self.current_adj = g0 self.current_stats = self.stats(self.current_adj) self.current_logweight = self.theta.dot(self.current_stats) self.proposed_stats = np.zeros_like(self.current_stats) # self.current_logweight = self.logweight(g0) log_msg("sample_gibbs: %8d nodes" % n_nodes, out=print_logs) log_msg("sample_gibbs: %8d burn-in steps" % burn_in, out=print_logs) log_msg("sample_gibbs: %8d steps between samples" % n_steps, out=print_logs) log_msg("sample_gibbs: %8d total steps" % total_steps, out=print_logs) log_msg("sample_gibbs: beginning MCMC process", out=print_logs) t_start = time.time() for step in range(total_steps): # assuming the logweight of the current state is already computed, we just need to compute the new values delta_k = self.delta_stats(self.current_adj, edge_sequence[0, step], edge_sequence[1, step]) self.proposed_stats[:] = self.current_stats[:] - delta_k[:] # the [:] are there to avoid new allocations(?) self.proposed_logweight = self.theta.dot(self.proposed_stats) p_flip = 1 / (1 + math.exp(self.theta.dot(delta_k))) if urand[step] < p_flip: # flip the edge, save the logweight and stats self._toggle_current_edge(edge_sequence[0, step], edge_sequence[1, step]) self.current_stats[:] = self.proposed_stats[:] self.current_logweight = self.proposed_logweight # avoid modifying self.current_adj, which may be sparse, until we're sure we're flipping the edge. if step >= burn_in and (step - burn_in) % n_steps == 0: # emit sample sample_num = (step - burn_in) // n_steps if print_logs is not None and (sample_num + 1) % (n_samples // 10) == 0: log_msg("sample_gibbs: emitting sample %8d / %8d" % (sample_num + 1, n_samples), out=print_logs) # samples[:, :, sample_num] = self.current_adj[:, :] if self.use_sparse: # samples[sample_num][:, :] = self.current_adj[:, :] samples[sample_num] = self.current_adj.copy() else: samples[:, :, sample_num] = self.current_adj[:, :] sample_stats[:, sample_num] = self.current_stats[:] t_end = time.time() log_msg("sample_gibbs: Sampling finished in", t_end - t_start, "s", out=print_logs) return samples, sample_stats
def ast_call(node): """Visit and transform ast call node""" distributed_mode = util_global.get_value("distributed_mode", "") is_not_strategy = distributed_mode in ("horovod", "") is_not_horovod = distributed_mode in ("tf_strategy", "") convert_loss_scale_api(node) if _call_name_match(node.func, "set_experimental_options"): log_msg( getattr(node, 'lineno', 'None'), 'change set_experimental_options(*) to set_experimental_options(experimental_options)' ) node.args = [ast.Name(id='experimental_options', ctx=ast.Load())] node.keywords = [] util_global.set_value('need_conver', True) if isinstance(node.func, ast.Name) and node.func.id == 'check_available_gpus': log_msg(getattr(node, 'lineno', 'None'), "change check_available_gpus() to ['/device:CPU:0']") util_global.set_value('need_conver', True) return ast.List(elts=[ast.Str(s="/device:CPU:0")], ctx=ast.Load()) if ((isinstance(node.func, ast.Name) and node.func.id == 'GraphOptions') or (isinstance(node.func, ast.Attribute) and node.func.attr == 'GraphOptions')): log_success_report(getattr(node, 'lineno', 'None'), 'GraphOptions()') src = copy.deepcopy(node) node.func = ast.Name(id='npu_graph_options', ctx=ast.Load()) node.args = [] node.keywords = [] node.keywords.append(ast.keyword(arg='graph_options', value=src)) util_global.set_value('need_conver', True) return node if (isinstance(node.func, ast.Name) and node.func.id == 'OptimizerOptions') or \ (isinstance(node.func, ast.Attribute) and node.func.attr == 'OptimizerOptions'): log_success_report(getattr(node, 'lineno', 'None'), 'OptimizerOptions()') src = copy.deepcopy(node) node.func = ast.Name(id='npu_optimizer_options', ctx=ast.Load()) node.args = [] node.keywords = [] node.keywords.append(ast.keyword(arg='optimizer_options', value=src)) util_global.set_value('need_conver', True) return node if _call_name_match(node.func, "Session"): return convert_origin_func_to_npu(node, tf_func_map["tf.Session"], "tf.Session", ["config"]) if _call_name_match(node.func, "InteractiveSession"): return convert_origin_func_to_npu(node, tf_func_map["tf.InteractiveSession"], "tf.InteractiveSession", ["config"]) if isinstance(node.func, ast.Attribute ) and node.func.attr == "BroadcastGlobalVariablesHook": if isinstance(node.func.value, ast.Name) and node.func.value.id == "hvd": if is_not_horovod: log_strategy_distributed_mode_error(node) return node log_msg( getattr(node, "lineno", "None"), 'change hvd.BroadcastGlobalVariablesHook to NPUBroadcastGlobalVariablesHook' ) node = pasta.parse( "NPUBroadcastGlobalVariablesHook(0, int(os.getenv('RANK_ID', '0')))" ) util_global.set_value('need_conver', True) return node if isinstance(node.func, ast.Attribute ) and node.func.attr == "BroadcastGlobalVariablesCallback": if isinstance(node.func.value, ast.Attribute) and node.func.value.attr == "callbacks": if is_not_horovod: log_strategy_distributed_mode_error(node) return node log_msg( getattr(node, "lineno", "None"), 'change hvd.callbacks.BroadcastGlobalVariablesCallback to NPUBroadcastGlobalVariablesCallback' ) node = pasta.parse( "NPUBroadcastGlobalVariablesCallback(root_rank=0)") util_global.set_value('need_conver', True) return node if isinstance(node.func, ast.Attribute) and node.func.attr == "DistributedOptimizer": if isinstance(node.func.value, ast.Name) and node.func.value.id == "hvd": if is_not_horovod: log_strategy_distributed_mode_error(node) return node return convert_hvd_distributed_api(node) if isinstance(node.func, ast.Attribute) and node.func.attr == 'shard': log_success_report(getattr(node, "lineno", "None"), 'shard') node.args = [ pasta.parse("int(os.getenv('RANK_SIZE', '1'))"), pasta.parse("int(os.getenv('RANK_ID', '0'))") ] node.keywords.clear() util_global.set_value('need_conver', True) return node if isinstance(node.func, ast.Attribute) and node.func.attr == 'dropout': if isinstance(node.func.value, ast.Attribute) and node.func.value.attr == 'nn': for index, _ in enumerate(node.args): if index == 2: return node for keyword in node.keywords: if keyword.arg == "noise_shape": return node log_success_report(getattr(node, "lineno", "None"), 'dropout') node.func = ast.Attribute(value=ast.Name(id='npu_ops', ctx=ast.Load()), attr='dropout', ctx=ast.Load()) keywords_new = [] for keyword in node.keywords: if keyword.arg != 'rate': keywords_new.append(keyword) else: keywords_new.append( ast.keyword(arg='keep_prob', value=ast.BinOp(left=ast.Num(n=1), op=ast.Sub(), right=keyword.value))) node.keywords = keywords_new util_global.set_value('need_conver', True) return node if isinstance(node.func, ast.Attribute) and \ ((node.func.attr == 'map_and_batch') or (node.func.attr == 'batch' and (not isinstance(node.func.value, ast.Attribute) or ( isinstance(node.func.value, ast.Attribute) and node.func.value.attr != 'train')))): exist = False for keyword in node.keywords: if keyword.arg == 'drop_remainder': exist = True if ((isinstance(keyword.value, ast.NameConstant) and not keyword.value.value) or (not isinstance(keyword.value, ast.NameConstant))): log_success_report(getattr(node, "lineno", "None"), node.func.attr) keyword.value = pasta.parse('True') util_global.set_value('need_conver', True) if not exist: log_success_report(getattr(node, "lineno", "None"), node.func.attr) keyword = ast.keyword(arg='drop_remainder', value=pasta.parse('True')) node.keywords.insert(0, keyword) util_global.set_value('need_conver', True) return node if (isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name) and node.func.value.id == 'tf' and node.func.attr == 'device'): log_success_report(getattr(node, "lineno", "None"), node.func.attr) node.args = [ast.Str(s='/cpu:0')] util_global.set_value('need_conver', True) return node if isinstance(node.func, ast.Attribute) and \ (node.func.attr == "get_distribution_strategy" or node.func.attr == "MirroredStrategy" or node.func.attr == "MultiWorkerMirroredStrategy"): if is_not_strategy: log_hvd_distributed_mode_error(node) return node log_success_report(getattr(node, "lineno", "None"), node.func.attr) new_func = ast.Attribute(value=ast.Name(id="npu_strategy", ctx=ast.Load()), attr="NPUStrategy", ctx=ast.Load()) ast.copy_location(new_func, node.func) node.func = new_func node.keywords = [] node.args = [] util_global.set_value('need_conver', True) return node if (isinstance(node.func, ast.Attribute) and (node.func.attr == 'RunConfig')) and \ (_call_name_match(node.func.value, 'estimator') or _call_name_match(node.func.value, 'tpu')): if node.keywords.count("train_distribute") or node.keywords.count( "eval_distribute"): if is_not_strategy: log_hvd_distributed_mode_error(node) save_summary_steps = None for keyword in node.keywords: if keyword.arg == 'save_summary_steps': save_summary_steps = keyword break if len(node.args) < 3 and not save_summary_steps: log_msg(getattr(node, 'lineno'), 'RunConfig() add save_summary_steps=0') util_global.set_value('need_conver', True) node.keywords.append( ast.keyword(arg='save_summary_steps', value=pasta.parse('0'))) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'TPUEstimator') and \ ((isinstance(node.func.value, ast.Attribute) and (node.func.value.attr == 'tpu')) or (isinstance(node.func.value, ast.Name) and (node.func.value.id == 'tpu'))): add_eval_on_tpu = True add_use_tpu = True add_export_to_tpu = True for keyword in node.keywords: if (keyword.arg == 'eval_on_tpu') or ( keyword.arg == 'use_tpu') or (keyword.arg == 'export_to_tpu'): if (not isinstance(keyword.value, ast.NameConstant)) or \ (isinstance(keyword.value, ast.NameConstant) and (keyword.value.value)): log_success_report(getattr(node, 'lineno', 'None'), 'TPUEstimator(' + keyword.arg + '=*)') keyword.value = pasta.parse('False') util_global.set_value('need_conver', True) if add_eval_on_tpu and (keyword.arg == 'eval_on_tpu'): add_eval_on_tpu = False if add_use_tpu and (keyword.arg == 'use_tpu'): add_use_tpu = False if add_export_to_tpu and (keyword.arg == 'export_to_tpu'): add_export_to_tpu = False if add_eval_on_tpu: log_success_report(getattr(node, 'lineno', 'None'), 'TPUEstimator(eval_on_tpu=*)') node.keywords.append( ast.keyword(arg='eval_on_tpu', value=pasta.parse('False'))) util_global.set_value('need_conver', True) if add_use_tpu: log_success_report(getattr(node, 'lineno', 'None'), 'TPUEstimator(use_tpu=*)') node.keywords.append( ast.keyword(arg='use_tpu', value=pasta.parse('False'))) util_global.set_value('need_conver', True) if add_export_to_tpu: log_success_report(getattr(node, 'lineno', 'None'), 'TPUEstimator(export_to_tpu=*)') node.keywords.append( ast.keyword(arg='export_to_tpu', value=pasta.parse('False'))) util_global.set_value('need_conver', True) if isinstance(node.func, ast.Attribute) and (node.func.attr == 'VirtualDeviceConfiguration'): log_success_report(getattr(node, 'lineno', 'None'), 'VirtualDeviceConfiguration') util_global.set_value('need_conver', True) memory_limit = None for keyword in node.keywords: if keyword.arg == 'memory_limit': memory_limit = keyword break if memory_limit: memory_limit.value = ast.NameConstant(value=None) else: node.keywords.append( ast.keyword(arg='memory_limit', value=ast.NameConstant(value=None))) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'set_soft_device_placement'): log_success_report(getattr(node, 'lineno', 'None'), 'set_soft_device_placement') util_global.set_value('need_conver', True) node.args = [] node.keywords = [ ast.keyword(arg='enabled', value=ast.NameConstant(value=True)) ] return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'set_memory_growth'): log_success_report(getattr(node, 'lineno', 'None'), 'set_memory_growth') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'set_virtual_device_configuration'): log_success_report(getattr(node, 'lineno', 'None'), 'set_virtual_device_configuration') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'jit_scope'): if isinstance(node.func.value, ast.Attribute) and (node.func.value.attr == 'experimental'): if isinstance(node.func.value.value, ast.Attribute) and (node.func.value.value.attr == 'xla'): log_success_report(getattr(node, 'lineno', 'None'), '*.xla.experimental.jit_scope') util_global.set_value('need_conver', True) compile_ops = None for keyword in node.keywords: if keyword.arg == 'compile_ops': compile_ops = keyword break if compile_ops: compile_ops.value = pasta.parse('False') else: node.keywords.append( ast.keyword(arg='compile_ops', value=pasta.parse('False'))) return node for estimator in util_global.get_value('Estimators', []): if (isinstance(node.func, ast.Attribute) and (node.func.attr == estimator)) \ or (isinstance(node.func, ast.Name) and (node.func.id == estimator)): log_msg( getattr(node, 'lineno'), "".join([estimator, '() add config=npu_run_config_init()'])) config = None for keyword in node.keywords: if keyword.arg == 'config': config = keyword break if config: new_value = ast.Call(func=ast.Name(id='npu_run_config_init', ctx=ast.Load()), args=[], keywords=[ ast.keyword(arg='run_config', value=config.value) ]) ast.copy_location(new_value, config.value) config.value = new_value else: node.keywords.append( ast.keyword(arg='config', value=pasta.parse('npu_run_config_init()'))) util_global.set_value('need_conver', True) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'clear_session'): log_msg(getattr(node, 'lineno'), "change keras.clear_session() to npu_clear_session()") node = ast.Call(func=ast.Name(id='npu_clear_session', ctx=ast.Load()), args=[], keywords=[]) util_global.set_value('need_conver', True) if _call_name_match(node.func, "MonitoredTrainingSession"): return convert_origin_func_to_npu( node, tf_func_map["tf.train.MonitoredTrainingSession"], "MonitoredTrainingSession", ["config", "hooks"]) if isinstance(node.func, ast.Attribute) and node.func.attr == "managed_session": return convert_origin_func_to_npu( node, tf_func_map["tf.train.Supervisor.managed_session"], "managed_session", ["config"], True) if distributed_mode == "tf_strategy": # this cond should be placed at the end of the Call function. return convert_distributed_strategy_apis(node) return node
def get_FlyBrain_graph(hemibrain_version="v1.1", directory="/notebooks/nx_graph_utils/hemibrain/exported-traced-adjacencies-", nodes_file="traced-neurons.csv", edges_file="traced-total-connections.csv", clusters_dir="/notebooks/nx_graph_utils/hemibrain/und_flybrain_v1.1/"): """Load the flybrain graph from the specified files. Returns a Networkx graph with weighted edges and labeled nodes. """ directory = directory + hemibrain_version log_msg("Attempting to read edges from", os.path.join(directory, edges_file)) edge_data = open(os.path.join(directory, edges_file), "r") next(edge_data, None) # skip the first line of the file, which is just a header row t_start = time.time() FlyBrain = nx.parse_edgelist(edge_data, delimiter=",", create_using=nx.DiGraph, nodetype=int, data=[("n_synapses", int)]) t_end = time.time() edge_data.close() log_msg("Read in {} nodes and {} edges in {} s:".format(nx.number_of_nodes(FlyBrain), nx.number_of_edges(FlyBrain), t_end - t_start)) print(nx.info(FlyBrain)) node_data = open(os.path.join(directory, nodes_file), 'r') # next(node_data,None) log_msg("Attempting to read cell type data from {}".format(os.path.join(directory, nodes_file))) line_counter = 0 for line in node_data.readlines()[1:]: l = line.split(",") ct = l[1].strip() inst = l[2].strip() if len(ct) == 0: ct = "None" if len(inst) == 0: inst = "None" nx.set_node_attributes(FlyBrain, {int(l[0]): {"celltype": ct, "instance": inst}}) # in v1.1 it's switched node_data.seek(0) n_clusters = {} cluster_hist = {} for filename in os.listdir(clusters_dir): log_msg("Reading file", filename) if "unweighted" in filename: cluster_name = "unweighted" elif "lr" in filename: cluster_name = "lr" else: try: cluster_name = (filename.split(".")[0]).split("_")[2] cluster_name = cluster_name[1:].replace("p", ".") except: log_msg("Error reading file", filename, ", skipping file") continue # lines_read, clust_histogram = read_cluster_ids(FlyBrain, filename, cluster_name) cluster_info = open(os.path.join(clusters_dir, filename), "r") for nl, cl in zip(node_data.readlines()[1:], cluster_info.readlines()): n_id = int(nl.split(",")[0]) c_id = int(cl) nx.set_node_attributes(FlyBrain, {n_id: {cluster_name: c_id}}) n_clusters[cluster_name] = clust_histogram.shape[0] cluster_hist[cluster_name] = clust_histogram node_data.seek(0) log_msg(n_clusters) node_data.close() return FlyBrain, n_clusters, cluster_hist
"""Testing the simplest version of the parameter estimation, using gradient ascent on the loglikelihood (using sample estimates of the ensemble mean) """ import sys import time import networkx as nx import numpy as np from ergm import ERGM from util import log_msg log_msg("BEGIN SCRIPT:", __file__) log_msg("Testing parameter estimation using sample mean of ensemble average") n_nodes = 25 n_observed = 1000 seed = 17289 np.random.seed(seed) max_edges = n_nodes * (n_nodes - 1) // 2 max_triangles = n_nodes * (n_nodes - 1) * (n_nodes - 2) // 6 theta_true = np.array([1.0, -10000.0]) # be fairly dense, but avoid triangles (cycles all length > 4) log_msg("Using ergm to produce", n_observed, "observations of low-triangle graphs") log_msg("theta =", theta_true) def edge_triangle(g): return np.array([g.sum() / (2 * max_edges), np.trace(g @ g @ g) / (6 * max_triangles)])