def convert_loss_scale_api(node):
"""Convert loss scale related Tensorflow APIs"""
if isinstance(node.func, ast.Attribute):
if node.func.attr == "FixedLossScale":
log_msg(
getattr(node, 'lineno', 'None'),
"change tf.train.experimental.FixedLossScale"
" to FixedLossScaleManager")
node.func = ast.Name(id="FixedLossScaleManager", ctx=ast.Load())
if len(node.keywords) == 1:
node.keywords[0].arg = "loss_scale"
util_global.set_value('need_conver', True)
return node
if node.func.attr == "DynamicLossScale":
return convert_dynamic_loss_scale(node)
if node.func.attr == "MixedPrecisionLossScaleOptimizer":
log_msg(
getattr(node, 'lineno', 'None'),
"change tf.train.experimental.MixedPrecisionLossScaleOptimizer"
" to NPULossScaleOptimizer")
node.func = ast.Name(id="NPULossScaleOptimizer", ctx=ast.Load())
for keyword in node.keywords:
if keyword.arg == "loss_scale":
keyword.arg = "loss_scale_manager"
if (len(util_global.get_value("distributed_mode", "")) != 0):
node.keywords.append(
ast.keyword(arg="is_distributed",
value=pasta.parse("True")))
util_global.set_value('need_conver', True)
return node
def import_from(node):
"""Modify node based on import module"""
if node.module:
values = node.module.split(".")
if "keras" in values:
util_global.set_value('is_keras_net', True)
if "horovod" in values:
log_msg(getattr(node, "lineno", "None"),
"remove horovod import line to None")
util_global.set_value('has_hvd_api', True)
new_node = ast.Expr(value=ast.NameConstant(value=None))
ast.copy_location(new_node, node)
util_global.set_value('need_conver', True)
return new_node
for value in node.names:
if isinstance(value, ast.alias):
values = value.name.split(".")
if "keras" in values:
util_global.set_value('is_keras_net', True)
if "horovod" in values:
log_msg(getattr(node, "lineno", "None"),
"remove horovod import line to None")
util_global.set_value('has_hvd_api', True)
new_node = ast.Expr(value=ast.NameConstant(value=None))
ast.copy_location(new_node, node)
util_global.set_value('need_conver', True)
return new_node
util_global.set_value('need_conver', True)
return node
def ast_if(node):
if isinstance(node.test, ast.Compare):
if len(node.test.comparators) == 1 and isinstance(
node.test.comparators[0], ast.Str):
if node.test.comparators[0].s == "__main__":
util_global.set_value("is_main_file", False)
util_global.set_value("has_main_func", True)
if util_global.get_value("is_keras_net", False):
log_msg(getattr(node, "lineno", "None"),
" add keras session npu config")
close_sess_call = ast.Call(
func=ast.Name(id="close_session", ctx=ast.Load()),
args=[ast.Name(id="npu_keras_sess", ctx=ast.Load())],
keywords=[])
keras_sess_assign = ast.Assign(
targets=[
ast.Name(id="npu_keras_sess", ctx=ast.Store())
],
value=ast.Call(func=ast.Name(
id="set_keras_session_npu_config", ctx=ast.Load()),
args=[],
keywords=[]))
node.body = [keras_sess_assign] + node.body + [
ast.Expr(value=close_sess_call)
]
util_global.set_value('need_conver', True)
if util_global.get_value("has_hccl_api", False):
log_msg(getattr(node, "lineno", "None"),
" add npu resource init api")
close_sess_call = ast.Call(
func=ast.Name(id="close_session", ctx=ast.Load()),
args=[ast.Name(id="npu_sess", ctx=ast.Load())],
keywords=[])
init_assign = ast.Assign(targets=[
ast.Tuple(elts=[
ast.Name(id="npu_sess", ctx=ast.Store()),
ast.Name(id="npu_shutdown", ctx=ast.Store())
],
ctx=ast.Store())
],
value=ast.Call(func=ast.Name(
id="init_resource",
ctx=ast.Load()),
args=[],
keywords=[]))
shutdown_call = ast.Call(func=ast.Name(
id="shutdown_resource", ctx=ast.Load()),
args=[
ast.Name(id="npu_sess",
ctx=ast.Load()),
ast.Name(id="npu_shutdown",
ctx=ast.Load())
],
keywords=[])
node.body = [init_assign] + node.body + [
ast.Expr(value=shutdown_call),
ast.Expr(value=close_sess_call)
]
util_global.set_value('need_conver', True)
return node
def imge_transfer(url,surl):
# post url to image downloader
pts = url.split("?")[0].strip("/").split("/")
name = "spider/%s" % "/".join(pts[2:])
nsrc = url_temp % name
util.log_msg("image_process\t%s\t%s\t%s" % (url,name,surl))
return (url,"/%s/%s" % (bucket_name,name),nsrc)
def create_post(post, msg, post_type, msg_type):
"""
function for posting to subreddit; separate post for long monster table
posts; self-edits signature for easy delete request PM link
"""
try:
for i, m in enumerate(msg):
if len(msg) > 1:
m = m.replace('___MTABLE___', "(%s of %s)" % (i + 1, len(msg)))
else:
m = m.replace('___MTABLE___', "")
if post_type == 'SUBMISSIONS':
if debug:
print m + signature_intro + signature
else:
c = post.add_comment(m + signature_intro + signature)
log_msg("Made a %s comment in %s" % (msg_type, post.short_link))
update_db(log_coll, stat_coll, msg_type, post.short_link, '')
elif post_type == 'COMMENTS':
if debug:
print m + signature_intro + signature
else:
c = post.reply(m + signature_intro + signature)
log_msg("Made a %s reply to %s" % (msg_type, post.permalink))
update_db(log_coll, stat_coll, msg_type, post.permalink, '')
sig_temp = signature_add.replace('___CID___', str(c.id))
sig_temp = sig_temp.replace('___PID___', str(c.link_id)[3:])
if not debug:
sleep(SLEEP)
m_tmp = c.body.replace('^^Processing...', sig_temp)
m_tmp = m_tmp.replace('&#', '&#')
r.get_info(thing_id='t1_' + str(c.id)).edit(m_tmp)
sleep(SLEEP_LONG)
except Exception as e:
log_error(e)
def get_info(content,info=None):
try:
p = {"collection_token":"","cursor":"","disablepager":False,"overview":False,"profile_id":"100008346345446","pagelet_token":"","tab_key":"friends","lst":"","ftid":None,"order":None,"sk":"friends","importer_state":None}
loader_info = re.findall('"enableContentLoader"([^\[]+\[[^\]]+){2}',content)
lst = None
pglt = None
if info is None:
page_token = re.findall('pagelet_token"?:[^\}]+}',content)
page_token = page_token[0]
lst = page_token.split("\"")[-2]
pglt = page_token[page_token.find(":\"")+2:].split("\"")[0]
else:
lst = info[0]
pglt = info[1]
loader_info = loader_info[0]
pfx1 = "pagelet_timeline_app_collection_"
idx1 = loader_info.find(pfx1)
ct = None
if idx1 > -1:
ct = loader_info[idx1+len(pfx1):].split("\"")[0]
cs = loader_info.split("\"")[-2]
np = p.copy()
np["collection_token"] = ct
np["cursor"] = cs
np["pagelet_token"] = pglt
np["lst"] = lst
np = json.dumps(np)
return (np,(lst,pglt))
except Exception:
tp, e,trace = sys.exc_info()
util.log_msg("parse paramerter error:"+str(e),"error")
return (None,None)
def convert_dynamic_loss_scale(node):
"""Convert dynamic loss scale related Tensorflow APIs"""
log_msg(
getattr(node, 'lineno', 'None'),
"change tf.train.experimental.DynamicLossScale"
" to ExponentialUpdateLossScaleManager")
node.func = ast.Name(id="ExponentialUpdateLossScaleManager",
ctx=ast.Load())
def check_arg(node):
initial_loss_scale = None
increment_period = None
multiplier = None
for index, arg in enumerate(node.args):
if index == 0:
initial_loss_scale = arg
if index == 1:
increment_period = arg
if index == 2:
multiplier = arg
for keyword in node.keywords:
if keyword.arg == "initial_loss_scale":
keyword.arg = "init_loss_scale"
initial_loss_scale = keyword
if keyword.arg == "increment_period":
keyword.arg = "incr_every_n_steps"
increment_period = keyword
if keyword.arg == "multiplier":
keyword.arg = "incr_ratio"
multiplier = keyword
return (initial_loss_scale, increment_period, multiplier)
(initial_loss_scale, increment_period, multiplier) = check_arg(node)
if initial_loss_scale:
if not isinstance(initial_loss_scale, ast.keyword):
node.keywords.append(
ast.keyword(arg="init_loss_scale", value=initial_loss_scale))
else:
node.keywords.append(
ast.keyword(arg="init_loss_scale", value=pasta.parse("2**15")))
if increment_period:
if not isinstance(increment_period, ast.keyword):
node.keywords.append(
ast.keyword(arg="incr_every_n_steps", value=increment_period))
else:
node.keywords.append(
ast.keyword(arg="incr_every_n_steps", value=pasta.parse("2000")))
if multiplier:
if not isinstance(multiplier, ast.keyword):
node.keywords.append(
ast.keyword(arg="incr_ratio", value=multiplier))
else:
node.keywords.append(
ast.keyword(arg="incr_ratio", value=pasta.parse("2")))
node.args = []
util_global.set_value('need_conver', True)
return node
def get_params(p):
try:
status,op = subprocess.getstatusoutput("node "+p)
if status == 0:
return op
else:
return None
except Exception:
tp, e,trace = sys.exc_info()
util.log_msg("execute javascript file error:%s" % str(e))
return None
def get_user_info(u,fp):
ourl = "http://www.toutiao.com/c/user/%s/" % u
url,html = util.get_page({"url":ourl})
p,h = get_urlinfo(url)
params = get_params("func.js")
uid = get_userinfo(html)
if params is not None and uid is not None:
params = json.loads(params)
params["user_id"] = uid
path = "/c/user/article/"
nurl = "%s//%s%s" % (p,h,path)
count = 3
while True:
url,html = util.get_page({"url":nurl,"data":params,"method":"post"})
if html is None or len(html) == 0:
util.log_msg("could not get data from url:%s,data:%s,uid:%s" % (nurl,str(params),u))
break
mp = json.loads(html)
if "data" in mp and isinstance(mp["data"],list):
if len(mp["data"]) == 0:
util.log_msg("no data from response.url:%s" % nurl)
result = []
for item in mp["data"]:
turl = util.parse_url(url,item["source_url"])
try:
get_article(turl,url,item,fp,result)
except Exception:
tp, e,trace = sys.exc_info()
util.log_msg("get article(url:%s) info error:%s" % (turl,str(e)))
if len(result) > 0:
if fp is None:
insert_into_db(result)
else:
for item in result:
fp.write("[%s]\t%s\t%s\n" % (time.ctime(),u,json.dumps(item)))
else:
util.log_msg("no data in content.url:%s" % nurl)
if mp["has_more"]:
params = get_params("func.js")
params = json.loads(params)
params["user_id"] = uid
nxt = mp["next"]
for key in nxt.keys():
params[key]=nxt[key]
else:
break
count -= 1
if count <= 0:
break
else:
util.log_msg("could not parse data from html file,need to check this out.url:%s,referer:%s." % (ourl,referer))
def delete_post(post, type, parent_url, user):
"""
function for deleting own post
"""
try:
if type == 'SCORE':
log_msg("Deleting post under %s due to downvote" % parent_url)
update_db(log_coll, stat_coll, 'DEL_SCORE', parent_url, user)
if type == 'PM':
log_msg("Deleting post under %s due to PM request from %s" % (parent_url, user))
update_db(log_coll, stat_coll, 'DEL_PM', parent_url, user)
post.delete()
sleep(SLEEP)
except Exception as e:
log_error(e)
def table_output(padx, msg):
"""
uses PADX data to build reddit post output; automatically creates a new
post if the # of chars > 10000 (reddit limit); automatically apends bot
signature
"""
table_header = "%s###### 	\n#### 	\n##### 	\n|||Expanded Monster Information ___MTABLE___ [hover to view]|\n:--:|--:|:--\n" % signature_intro
if len(msg) == 0:
msg.append(table_header)
i = len(msg) - 1
sid = int(padx.id) / 200
cid = int(padx.id) % 200
msg_temp = "[](#I/S%02d/C%03d/%s \"%s\")|" % (sid, cid, padx.id, padx.name)
msg_temp = "%s#%s|**[%s](http://www.puzzledragonx.com/en/monster.asp?n=%s)**\n" % (msg_temp, padx.id, padx.name, padx.id)
if len(padx.type_formatted) > 0:
msg_temp = "%s%s" % (msg_temp, padx.type_formatted[0])
else:
msg_temp = "%s " % msg_temp
if padx.ls_id != '0':
msg_temp = "%s|Leader|**[%s](http://www.puzzledragonx.com/en/leaderskill.asp?s=%s)**: %s\n" % (msg_temp, padx.ls_name, padx.ls_id, padx.ls_text)
else:
msg_temp = "%s|Leader|**%s**\n" % (msg_temp, padx.ls_name)
if len(padx.type_formatted) > 1:
msg_temp = "%s%s" % (msg_temp, padx.type_formatted[1])
else:
msg_temp = "%s " % msg_temp
if padx.as_id != '0':
msg_temp = "%s |Active|**[%s](http://www.puzzledragonx.com/en/skill.asp?s=%s)**: %s (%s)\n" % (msg_temp, padx.as_name, padx.as_id, padx.as_text, padx.acd_text)
else:
msg_temp = "%s |Active|**%s**\n" % (msg_temp, padx.as_name)
if len(padx.awk) != 0:
awk_temp = ""
for a in padx.awk:
awk_temp = "%s[](#I/misc/C%03d/) " % (awk_temp, a)
msg_temp = "%s |Awoken|%s\n" % (msg_temp, awk_temp)
else:
msg_temp = "%s |Awoken|None\n" % (msg_temp)
if len(msg[i]) + len(msg_temp) + len(signature) + len(signature_add) > MSG_LEN_LIMIT:
log_msg("message long... splitting")
update_db(log_coll, stat_coll, 'SPLIT', '', '')
msg.append(table_header)
msg[i + 1] = msg[i + 1] + msg_temp
else:
msg[i] = msg[i] + msg_temp
return msg
def insert_keras_dropout_import(r_node):
"""Add keras dropout import module"""
npu_alias = ast.alias(name='npu_convert_dropout', asname=None)
npu_import = ast.ImportFrom(module='npu_bridge.estimator.npu',
names=[npu_alias],
level=0)
n = 0
lenline = len(r_node.body)
while n < lenline and not isinstance(r_node.body[n],
ast.ImportFrom) and not isinstance(
r_node.body[n], ast.Import):
n += 1
while n < lenline and (isinstance(r_node.body[n], ast.ImportFrom)):
n += 1
r_node.body.insert(n, npu_import)
log_msg(n, "from npu_bridge.estimator.npu import npu_convert_dropout")
def add_npu_func_to_params(node, param_index, org_func_name, param_name,
npu_func, npu_func_args):
"""Add npu function to parameters"""
param_node = None
if ((not util_global.get_value("distributed_mode", "")
or util_global.get_value("distributed_mode", "") == "horovod")
and (param_name in ("callbacks", "hooks", "optimizer"))):
return node
log_param_msg = "".join([org_func_name, " add npu ", param_name])
log_msg(getattr(node, "lineno", "None"), log_param_msg)
for index, _ in enumerate(node.args):
if param_index is not None and index == param_index:
param_node = node.args.pop(param_index)
for keyword in node.keywords:
if keyword.arg == param_name:
param_node = keyword
if param_node:
if isinstance(param_node, ast.keyword):
new_value = ast.Call(func=ast.Name(id=npu_func, ctx=ast.Load()),
args=[],
keywords=[
ast.keyword(arg=npu_func_args,
value=param_node.value)
])
ast.copy_location(new_value, param_node.value)
param_node.value = new_value
else:
node.keywords.append(
ast.keyword(arg=param_name,
value=ast.Call(func=ast.Name(id=npu_func,
ctx=ast.Load()),
args=[],
keywords=[
ast.keyword(arg=npu_func_args,
value=param_node)
])))
else:
node.keywords.append(
ast.keyword(arg=param_name,
value=pasta.parse("".join([npu_func, "()"]))))
return node
def insert_into_db(mps):
columns = ['id', 'title', 'source', 'a_type', 'username', 'description', 'content', 'source_url', 'publish_time', 'category', 'tags', 'cover', 'images', 'views', 'comments']
keys = ['id', 'title', 'from', ('isOriginal',1), 'source', 'abstract', 'content', 'url', 'publishtime', 'category', 'tags', 'cover', 'images', ('views',1), ('comments',1)]
tt = [("'{%s}'" % o) if isinstance(o,str) else ("{%s}" % o[0]) for o in keys]
table = "tb_article_crawl"
c2k_mp = {o[0]:o[1] for o in zip(columns,tt)}
update_columns = ["views","comments"]
data = []
for mp in mps:
mp["isOriginal"] = 1 if mp["isOriginal"] else 2
mp["from"] = "今日头条"
np = {k:(re.sub("[\r\n\t ]+"," ",mp[k]).replace("'","\\'") if isinstance(mp[k],str) else mp[k]) for k in mp.keys()}
temp_temp = "(%s) %s" % (",".join(tt), "on duplicate key update "+",".join(["`%s`=%s" % (o,c2k_mp[o]) for o in update_columns]) if len(update_columns)>0 else "")
try:
data.append(temp_temp.format_map(np))
except Exception:
tp, e,trace = sys.exc_info()
util.log_msg("format data error:%s" % str(e))
db_dump.dump_db(table,columns,data)
def replace_image(content,referer):
tree = etree.HTML(content)
nodes = tree.xpath("//img")
imgs = []
for node in nodes:
keys = node.keys()
key = "src" if "src" in keys else ("href" if "href" in keys else None)
if key is not None:
src = util.parse_url(referer,node.get(key))
if src.find("?") > -1:
# not good here
util.log_msg("expecting image url with no params%s" % src)
continue
_,name,nsrc = imge_transfer(src,referer)
node.set(key,nsrc)
imgs.append((src,name))
target = "".join([etree.tounicode(node) for node in tree.xpath("/html/body/*")])
imgs_str = "|".join([o[1] for o in imgs])
return target,imgs_str
def convert_hvd_distributed_api(node):
"""Convert horovod distributed APIs"""
log_msg(
getattr(node, "lineno", "None"),
'change hvd.DistributedOptimizer to npu_distributed_optimizer_wrapper')
node.func = ast.Name(id="npu_distributed_optimizer_wrapper",
ctx=ast.Load())
opt_keyword = None
for keyword in node.keywords:
if keyword.arg == "optimizer":
opt_keyword = keyword
node.keywords.clear()
if opt_keyword is None:
opt_arg = node.args[0]
node.args.clear()
node.args.append(opt_arg)
else:
node.keywords.append(opt_keyword)
util_global.set_value('need_conver', True)
return node
def load_cluster_info(G=None, clusters_dir="/notebooks/nx_graph_utils/hemibrain/und_flybrain_v1.1/"):
"""Loads cluster info into the provided graph. If no graph is specified, loads the flybrain data with
`get_FlyBrain_graph`. Returns the graph as well as two dictionaries with information about the clusters. """
n_clusters = {}
cluster_hist = {}
if G is None:
G = get_FlyBrain_graph()
for filename in os.listdir(clusters_dir):
log_msg("Reading file", filename)
if "unweighted" in filename:
cluster_name = "unweighted"
elif "lr" in filename:
cluster_name = "lr"
else:
try:
cluster_name = (filename.split(".")[0]).split("_")[2]
cluster_name = cluster_name[1:].replace("p", ".")
except:
log_msg("Error reading file", filename, ", skipping file")
continue
lines_read, clust_histogram = read_cluster_ids(G, filename, cluster_name)
n_clusters[cluster_name] = clust_histogram.shape[0]
cluster_hist[cluster_name] = clust_histogram
log_msg(n_clusters)
return G, n_clusters, cluster_hist
def insert_npu_import(r_node):
"""Add NPU import modules"""
npu_alias = ast.alias(name='*', asname=None)
npu_import = ast.ImportFrom(module='npu_bridge.npu_init',
names=[npu_alias],
level=0)
num = 5 if len(r_node.body) >= 5 else len(r_node.body)
import_index = 0
is_insert = False
for i in range(0, num):
if isinstance(r_node.body[i], ast.Import):
r_node.body.insert(i, npu_import)
log_msg(i, "from npu_bridge.npu_init import *")
is_insert = True
break
if isinstance(r_node.body[i], ast.ImportFrom):
if r_node.body[i].module != "__future__":
r_node.body.insert(i, npu_import)
log_msg(i, "from npu_bridge.npu_init import *")
is_insert = True
break
import_index = i + 1
if not is_insert:
r_node.body.insert(import_index, npu_import)
log_msg(import_index, "from npu_bridge.npu_init import *")
def get_article(url,referer,data,fp,result2):
url, h = util.get_page({"url":url,"headers":{"Referer":referer}})
tree = etree.HTML(h)
scripts = [o for o in tree.xpath("//script/text()") if o.find("BASE_DATA") > -1 or o.find("__pgcInfo")>-1]
scripts.append("console.log(JSON.stringify(BASE_DATA))")
open("data/tmp.js","w").write("\n".join(scripts))
r = get_params("data/tmp.js")
if r is not None:
mp = json.loads(r)
obj = {"entry":data,"data":mp}
conf = [("title",["data","artilceInfo","title"]),
("content",["data","artilceInfo","content"],None,html.unescape),
("comments",["data","commentInfo","comments_count"],0),
("isOriginal",["data","artilceInfo","subInfo","isOriginal"],False),
("url",["__const",url]),
("views",["entry","go_detail_count"], 0),
("cover",["entry","image_url"],""),
("abstract",["entry","abstract"], ""),
("source",["data","artilceInfo","subInfo","source"],""),
("publishtime",["data","artilceInfo","subInfo","time"]),
("tags",["data","artilceInfo","tagInfo","tags"],"",lambda o:",".join([so["name"] for so in o])),
("category",["data","headerInfo","chineseTag"],""),
]
result = {}
for cf in conf:
v = util.get_jpath(obj,cf[1],cf[2] if len(cf)>2 else None,cf[3] if len(cf)>3 else None)
if v is not None:
result[cf[0]] = v
result["id"] = hashlib.md5(url.encode("utf-8")).hexdigest()
if "content" in result:
result["content"],result["images"] = replace_image(result["content"],url)
if "cover" in result and len(result["cover"])>0:
result["cover"] = imge_transfer(util.parse_url(url,result["cover"]),url)[1]
if len(result) > 0:
result2.append(result)
else:
util.log_msg("could not parse content from html file,need to check this out.url:%s,referer:%s." % (url,referer))
else:
util.log_msg("could not parse data from html file,need to check this out.url:%s,referer:%s." % (url,referer))
def convert_distributed_strategy_apis(node):
"""Convert distributed strategy API"""
if isinstance(node.func, ast.Attribute) and isinstance(
node.func.value, ast.Attribute):
if ("Optimizer" in node.func.attr
and node.func.attr != "ScipyOptimizerInterface"
and node.func.attr != "MixedPrecisionLossScaleOptimizer"):
log_msg(getattr(node, "lineno", "None"),
"add npu distribute optimizer to tensorflow optimizer")
new_node = ast.Call(func=ast.Name(
id="npu_distributed_optimizer_wrapper", ctx=ast.Load()),
args=[node],
keywords=[])
ast.copy_location(new_node, node)
util_global.set_value('need_conver', True)
return new_node
if isinstance(
node.func, ast.Name
) and "Optimizer" in node.func.id and node.func.id != "NPULossScaleOptimizer":
log_msg(getattr(node, "lineno", "None"),
"add npu distribute optimizer to tensorflow optimizer")
new_node = ast.Call(func=ast.Name(
id="npu_distributed_optimizer_wrapper", ctx=ast.Load()),
args=[node],
keywords=[])
ast.copy_location(new_node, node)
util_global.set_value('need_conver', True)
return new_node
if _call_name_match(node.func, "TrainSpec"):
return convert_origin_func_to_npu(
node, tf_func_map["tf.estimator.TrainSpec"], "TrainSpec",
["hooks"])
if _call_name_match(node.func, "EvalSpec"):
return convert_origin_func_to_npu(node,
tf_func_map["tf.estimator.EvalSpec"],
"EvalSpec", ["hooks"])
if isinstance(node.func, ast.Attribute) and node.func.attr == "train":
if isinstance(node.func.value,
ast.Attribute) and node.func.value.attr == "learning":
return node
return convert_origin_func_to_npu(
node, tf_func_map["tf.estimator.Estimator.train"],
"Estimator.train", ["hooks"], True)
if isinstance(node.func, ast.Attribute) and (node.func.attr == 'compile'):
if isinstance(node.func.value,
ast.Name) and node.func.value.id == "re":
return node
return convert_origin_func_to_npu(
node, tf_func_map["tf.keras.Model.compile"], "Model.compile",
["optimizer"], True)
if isinstance(node.func, ast.Attribute) and node.func.attr == "fit":
return convert_origin_func_to_npu(node,
tf_func_map["tf.keras.Model.fit"],
"Model.fit", ["callbacks"], True)
if isinstance(node.func,
ast.Attribute) and node.func.attr == "fit_generator":
return convert_origin_func_to_npu(
node, tf_func_map["tf.keras.Model.fit_generator"],
"Model.fit_generator", ["callbacks"], True)
if isinstance(node.func, ast.Attribute) and node.func.attr == "gradients" and \
isinstance(node.func.value, ast.Name) and node.func.value.id == "tf":
return convert_tf_gradient_distributed(node)
return node
"""Testing the functionality of the ergm package"""
import sys
import time
import numpy as np
import networkx as nx
from ergm import ERGM
from util import log_msg
from scipy.special import binom
log_msg("BEGIN SCRIPT:", __file__)
log_msg("Testing ergms with adjacency matrices")
p0 = 0.1 # edge density for ER graph
n_nodes = 6
n_samples = 10000
seed = 17289 # Not actually used at the moment
np.random.seed(seed)
m = int(binom(n_nodes, 2))
log_msg("Using networkx to sample Erdos-Renyi random graphs with edge probability p = {}".format(p0))
log_msg("Producing {} samples with {} nodes".format(n_samples, n_nodes))
log_msg("Using seed {} for numpy; using 17 * k for nx random graphs".format(seed))
nx_ER_start = time.time()
nx_ER_list = [nx.gnp_random_graph(n_nodes, p0, seed=17 * k) for k in range(n_samples)]
nx_ER_end = time.time()
def _MLE_sampler(self,
observed,
n_estim_samples=100,
alpha=1,
alpha_rate=0.999,
max_iter=1000,
x_tol=1e-8,
print_logs=None,
sampler_logs=None,
**kwargs):
"""
Compute the maximum likelihood estimate (MLE) of parameters for the observed data using gradient ascent on
the likelihood. The expected value of the current ERGM is estimated by sampling.
:param observed: Observed graphs, shape n_nodes x n_nodes x n_samples
:param n_estim_samples: samples to use at each estimation step
:param alpha: Scale factor of gradient.
:param max_iter: Maximum number of iterations to take
:param x_tol: Quit trying to optimize if gradient norm falls below x_tol
:param print_logs: File pointer, where to print logs (default None to suppress output)
:param sampler_logs: File pointer, where sampler method should print logs (default None to suppress output)
:param kwargs: Additional keyword arguments are passed to sampler
:return: Dictionary including trajectory and such.
"""
if len(observed.shape) == 2:
k_obs = self.stats(observed)
log_msg("MLE_sampler: Passed single graph; observed stats:\n",
k_obs,
out=print_logs)
else:
try:
all_obs_k = np.array([
self.stats(observed[:, :, i])
for i in range(observed.shape[2])
])
except:
# if sparse arrays were used, observed will be an array of sparse matrices
all_obs_k = np.array([
self.stats(observed[i].tocsr())
for i in range(observed.shape[0])
])
k_obs = all_obs_k.mean(axis=0)
log_msg("MLE_sampler: Computed stats, resulting shape:",
all_obs_k.shape,
out=print_logs)
log_msg("MLE_sampler: average stats:", k_obs, out=print_logs)
log_msg("MLE_sampler: %8d estimate samples" % n_estim_samples,
out=print_logs)
log_msg("MLE_sampler: %8f alpha" % alpha, out=print_logs)
log_msg("MLE_sampler: %8d max iterations" % max_iter, out=print_logs)
log_msg("MLE_sampler: %8e L tolerance" % x_tol, out=print_logs)
# trajectory = np.zeros((max_iter, self.theta.shape[0]))
n_nodes = observed.shape[0]
# trajectory of thetas and such
theta_t = np.zeros((max_iter + 1, *self.theta.shape))
k_bar_t = np.zeros_like(theta_t)
grad_t = np.zeros_like(theta_t)
covar_t = np.zeros(
(max_iter + 1, *self.theta.shape, *self.theta.shape))
# local variables of the algorithm
delta_theta = np.zeros_like(self.theta)
theta_t[0, :] = self.theta # store initial value of theta
# the sample values at each iteration
# The actual sample graphs are not needed, so for now, I'll just ignore them. If they turn out to be useful,
# we can sort out how to handle sparse vs. non-sparse ERGMs.
# if self.use_sparse:
# G_samp = np.array([sparse.lil_matrix(0) for _ in range(n_estim_samples)])
# else:
# G_samp = np.zeros((n_nodes, n_nodes, n_estim_samples), dtype=int)
k_samp = np.zeros((*self.theta.shape, n_estim_samples))
log_msg(
f"{'iter':4} {'|theta(t)|':20} {'|gradient|':20} {'alpha':20} {'|Delta theta|':20}",
out=print_logs)
stopping_criteria = []
stop_iter = -1
for step in range(max_iter):
try:
_, k_samp[:, :] = self.sample_gibbs(n_nodes,
n_estim_samples,
print_logs=sampler_logs,
**kwargs)
k_bar_t[step, :] = k_samp.mean(axis=1)
covar_t[step, :, :] = np.cov(k_samp)
grad_t[step, :] = k_obs - k_bar_t[step, :]
grad_norm = np.linalg.norm(grad_t[step, :])
delta_theta[:] = alpha * grad_t[step, :]
log_msg(
f"{step:4d} {np.linalg.norm(theta_t[step, :]):20.8f} {grad_norm:20.8f} {alpha:20.8f} {alpha * grad_norm:20.8f}",
out=print_logs)
theta_t[step + 1, :] = theta_t[step, :] + delta_theta
self._set_theta(theta_t[step + 1, :], True)
# update alpha
alpha = alpha * alpha_rate
# check stopping criteria
# TODO implement 2nd order stopping criteria
if step + 1 == max_iter:
stopping_criteria.append("max_iter reached")
if grad_norm < x_tol:
stopping_criteria.append("grad_norm < x_tol")
if len(stopping_criteria) > 0:
stop_iter = step
# break
except KeyboardInterrupt:
stopping_criteria.append("keyboard interrupt")
stop_iter = step - 1
# break
except Exception as e:
stopping_criteria.append("unhandled exception: {}".format(e))
stop_iter = step - 1
# break
finally:
if len(stopping_criteria) > 0:
if stop_iter < 0:
stop_iter = step
break
trajectory = {
"theta": theta_t[:stop_iter, :],
"expected stats": k_bar_t[:stop_iter, :],
"covariances": covar_t[:stop_iter, :, :],
"gradient": grad_t[:stop_iter, :],
"stop": stopping_criteria,
"stop_iter": stop_iter
}
return trajectory
def check_posts(posts, post_type, forced):
"""
main function for checking submissions and replies made in subreddit;
checks for iconified monster links and 'flair in ID' messages
"""
for post in posts:
called = False
# skip if post made by bot
if str(post.author) == USERNAME:
continue
# check processed posts and ignore lists
if forced is False:
if post_type == 'SUBMISSIONS':
if str(post.id) in processed_submissions:
continue
if str(post.id) in ignored_submissions:
continue
elif post_type == 'COMMENTS':
if str(post.id) in processed_comments:
continue
if str(post.link_id)[3:] in ignored_submissions:
continue
if post_type == 'SUBMISSIONS':
temp_text = post.selftext.encode('utf-8')
elif post_type == 'COMMENTS':
temp_text = post.body.encode('utf-8')
# check callouts
if temp_text.find('-/u/tamabot') > -1:
log_msg('Found negative callout')
pass
elif temp_text.find('/u/tamabot') > -1:
log_msg('Found callout request!')
called = True
# check for Monster Icons - disabled
# if called is True or forced is True:
# n, temp_id, msg, listed = 0, 0, [], []
# for m in pattern_icon.finditer(temp_text):
# e = m.groupdict()
# if e['id'] is not None:
# temp_id = int(e['id'])
# elif e['sid'] is not None and e['cid'] is not None:
# temp_id = (int(e['sid'][1:]) * 50) + int(e['cid'][1:]) + 1
# n, listed, msg = process_monsters(temp_id, n, listed, msg)
# # check for PADX Team
# m = re.findall(pattern_padxsim, temp_text, re.I | re.U)
# if m:
# for e in m:
# for i in e:
# temp_id = int(i)
# n, listed, msg = process_monsters(temp_id, n, listed, msg)
# # create Monster Table
# if n > 0:
# create_post(post, msg, post_type, 'MONSTER TABLE')
# check for Flair Call
m = re.search(pattern_flair_call, temp_text, re.I | re.U)
if m or called is True or forced is True:
msg = []
if post.author_flair_text is not None:
msg.append("Found %s's flair: **%s**\n" % (str(post.author), post.author_flair_text))
create_post(post, msg, post_type, 'FLAIR ID')
# update processed posts list
if post_type == 'SUBMISSIONS':
check_processed_posts(processed_submissions, str(post.id), LIMIT)
elif post_type == 'COMMENTS':
check_processed_posts(processed_comments, str(post.id), LIMIT)
sleep(SLEEP_LONG)
def check_pm(msgs):
"""
function for checking bot's private messages; delete if parent commentor
(or subreddit moderator) has requested post deletion; stops bot if
moderator has requested a halt
"""
for msg in msgs:
# check for delete request
m = re.search(ur'^\+delete\s(.+?)$', msg.body.lower())
if m:
id = "t1_%s" % m.group(1)
c = r.get_info(thing_id = id)
if c is not None:
c_parent = r.get_info(thing_id = c.parent_id)
if c_parent.author is None:
delete_post(c, 'PM', '', msg.author.name)
else:
if msg.author.name == c_parent.author.name or msg.author.name in mod_list or msg.author.name == 'mrmin123':
if "Please request this post to be deleted to un-ignore." in c.body:
log_msg("Un-ignoring posts under %s by %s's request" % (c_parent.permalink, msg.author.name))
update_db(log_coll, stat_coll, 'UNIGNORE', c_parent.permalink, msg.author.name)
try:
ignored_submissions.remove(str(c.parent_id)[3:])
except Exception as e:
log_error(e)
delete_post(c, 'PM', c_parent.permalink, msg.author.name)
else:
log_warning("Incorrect delete request from %s for %s" % (msg.author.name, c.permalink))
update_db(log_coll, stat_coll, 'DEL_BAD', c.permalink, msg.author.name)
# check for ignore request
m = re.search(ur'^\+ignore\s(.+?)$', msg.body.lower())
if m:
id = "t3_%s" % m.group(1)
c = r.get_info(thing_id = id)
if c.author is not None and (msg.author.name == c.author.name or msg.author.name in mod_list or msg.author.name == 'mrmin123'):
if m.group(1) not in ignored_submissions:
ignored_submissions.append(m.group(1))
log_msg("Ignoring posts under %s by %s's request" % (c.short_link, msg.author.name))
update_db(log_coll, stat_coll, 'IGNORE_PM', c.short_link, msg.author.name)
temp_msg = "%s\nI am ignoring any new posts in this thread by OP/moderator's request! Please request this post to be deleted to un-ignore.\n" % signature_intro
create_post(c, [temp_msg], 'SUBMISSIONS', 'IGNORE_POST')
else:
if type(c) is praw.objects.Submission:
tempLink = c.short_link
elif type(c) is praw.objects.Comment:
tempLink = c.permalink
else:
tempLink = m.group(1)
log_warning("Incorrect ignore request from %s for %s" % (msg.author.name, tempLink))
update_db(log_coll, stat_coll, 'IGNORE_BAD', tempLink, msg.author.name)
# check for revisit
m = re.search(ur'^\+visit\s(.+?)$', msg.body.lower())
if m:
temp_type = 'SUBMISSIONS'
id = "t3_%s" % m.group(1)
c = r.get_info(thing_id = id)
if c is None:
temp_type = 'COMMENTS'
id = id = "t1_%s" % m.group(1)
c = r.get_info(thing_id = id)
if c is not None and c.subreddit.display_name.lower() == SUBREDDIT.lower() and (msg.author.name == c.author.name or msg.author.name in mod_list or msg.author.name == 'mrmin123'):
log_msg("Revisiting %s under %s's request" % (c.permalink, msg.author.name))
update_db(log_coll, stat_coll, 'REVISIT', c.permalink, msg.author.name)
check_posts([c], temp_type, True)
else:
if type(c) is praw.objects.Submission:
tempLink = c.short_link
elif type(c) is praw.objects.Comment:
tempLink = c.permalink
else:
tempLink = m.group(1)
log_msg("Incorrect revisit request for %s by %s" % (tempLink, msg.author.name))
update_db(log_coll, stat_coll, 'REVISIT_BAD', tempLink, msg.author.name)
# check for moderator halt request
if msg.author.name in mod_list or msg.author.name == 'mrmin123':
m = re.search(ur'^\+halt$', msg.body.lower())
if m:
msg.mark_as_read()
log_error("Bot halt requested by %s" % msg.author.name)
update_db(log_coll, stat_coll, 'HALT', '', msg.author.name)
exit()
msg.mark_as_read()
sleep(SLEEP)
sleep(SLEEP_LONG)
def convert_loss_scale_api(node):
if isinstance(node.func, ast.Attribute):
if node.func.attr == "FixedLossScale":
log_msg(
getattr(node, 'lineno', 'None'),
"change tf.train.experimental.FixedLossScale"
" to FixedLossScaleManager")
node.func = ast.Name(id="FixedLossScaleManager", ctx=ast.Load())
if len(node.keywords) == 1:
node.keywords[0].arg = "loss_scale"
util_global.set_value('need_conver', True)
return node
if node.func.attr == "DynamicLossScale":
log_msg(
getattr(node, 'lineno', 'None'),
"change tf.train.experimental.DynamicLossScale"
" to ExponentialUpdateLossScaleManager")
node.func = ast.Name(id="ExponentialUpdateLossScaleManager",
ctx=ast.Load())
initial_loss_scale = None
increment_period = None
multiplier = None
for index, arg in enumerate(node.args):
if index == 0: initial_loss_scale = arg
if index == 1: increment_period = arg
if index == 2: multiplier = arg
for keyword in node.keywords:
if keyword.arg == "initial_loss_scale":
keyword.arg = "init_loss_scale"
initial_loss_scale = keyword
if keyword.arg == "increment_period":
keyword.arg = "incr_every_n_steps"
increment_period = keyword
if keyword.arg == "multiplier":
keyword.arg = "incr_ratio"
multiplier = keyword
if initial_loss_scale:
if not isinstance(initial_loss_scale, ast.keyword):
node.keywords.append(
ast.keyword(arg="init_loss_scale",
value=initial_loss_scale))
else:
node.keywords.append(
ast.keyword(arg="init_loss_scale",
value=pasta.parse("2**15")))
if increment_period:
if not isinstance(increment_period, ast.keyword):
node.keywords.append(
ast.keyword(arg="incr_every_n_steps",
value=increment_period))
else:
node.keywords.append(
ast.keyword(arg="incr_every_n_steps",
value=pasta.parse("2000")))
if multiplier:
if not isinstance(multiplier, ast.keyword):
node.keywords.append(
ast.keyword(arg="incr_ratio", value=multiplier))
else:
node.keywords.append(
ast.keyword(arg="incr_ratio", value=pasta.parse("2")))
node.args = []
util_global.set_value('need_conver', True)
return node
if node.func.attr == "MixedPrecisionLossScaleOptimizer":
log_msg(
getattr(node, 'lineno', 'None'),
"change tf.train.experimental.MixedPrecisionLossScaleOptimizer"
" to NPULossScaleOptimizer")
node.func = ast.Name(id="NPULossScaleOptimizer", ctx=ast.Load())
for keyword in node.keywords:
if keyword.arg == "loss_scale":
keyword.arg = "loss_scale_manager"
util_global.set_value('need_conver', True)
return node
def ast_call(node):
convert_loss_scale_api(node)
if _call_name_match(node.func, "set_experimental_options"):
log_msg(
getattr(node, 'lineno', 'None'),
'change set_experimental_options(*) to set_experimental_options(experimental_options)'
)
node.args = [ast.Name(id='experimental_options', ctx=ast.Load())]
node.keywords = []
util_global.set_value('need_conver', True)
if isinstance(node.func,
ast.Name) and node.func.id == 'check_available_gpus':
log_msg(getattr(node, 'lineno', 'None'),
"change check_available_gpus() to ['/device:CPU:0']")
util_global.set_value('need_conver', True)
return ast.List(elts=[ast.Str(s="/device:CPU:0")], ctx=ast.Load())
if (isinstance(node.func, ast.Name) and node.func.id == 'ConfigProto') or \
(isinstance(node.func, ast.Attribute) and node.func.attr == 'ConfigProto'):
log_success_report(getattr(node, 'lineno', 'None'), 'ConfigProto()')
src = copy.deepcopy(node)
node.func = ast.Name(id='npu_config_proto', ctx=ast.Load())
node.args = []
node.keywords = []
node.keywords.append(ast.keyword(arg='config_proto', value=src))
util_global.set_value('need_conver', True)
return node
if (isinstance(node.func, ast.Name) and node.func.id == 'GraphOptions') or \
(isinstance(node.func, ast.Attribute) and node.func.attr == 'GraphOptions'):
log_success_report(getattr(node, 'lineno', 'None'), 'GraphOptions()')
src = copy.deepcopy(node)
node.func = ast.Name(id='npu_graph_options', ctx=ast.Load())
node.args = []
node.keywords = []
node.keywords.append(ast.keyword(arg='graph_options', value=src))
util_global.set_value('need_conver', True)
return node
if (isinstance(node.func, ast.Name) and node.func.id == 'OptimizerOptions') or \
(isinstance(node.func, ast.Attribute) and node.func.attr == 'OptimizerOptions'):
log_success_report(getattr(node, 'lineno', 'None'),
'OptimizerOptions()')
src = copy.deepcopy(node)
node.func = ast.Name(id='npu_optimizer_options', ctx=ast.Load())
node.args = []
node.keywords = []
node.keywords.append(ast.keyword(arg='optimizer_options', value=src))
util_global.set_value('need_conver', True)
return node
if (isinstance(node.func, ast.Name) and node.func.id == 'Session') or \
(isinstance(node.func, ast.Attribute) and node.func.attr == 'Session'):
log_success_report(getattr(node, 'lineno', 'None'), 'Session()')
config = None
for index, _ in enumerate(node.args):
if index == 2:
config = node.args.pop(2)
break
for keyword in node.keywords:
if keyword.arg == 'config':
config = keyword
break
if config:
if isinstance(config, ast.keyword):
config.value = ast.Call(func=ast.Name(id='npu_config_proto',
ctx=ast.Load()),
args=[],
keywords=[
ast.keyword(arg='config_proto',
value=config.value)
])
else:
node.keywords.append(
ast.keyword(arg='config',
value=ast.Call(
func=ast.Name(id='npu_config_proto',
ctx=ast.Load()),
args=[],
keywords=[
ast.keyword(arg='config_proto',
value=config)
])))
else:
node.keywords.append(
ast.keyword(arg='config',
value=ast.Call(func=ast.Name(id='npu_config_proto',
ctx=ast.Load()),
args=[],
keywords=[])))
util_global.set_value('need_conver', True)
return node
if isinstance(node.func, ast.Attribute
) and node.func.attr == "BroadcastGlobalVariablesHook":
log_success_report(getattr(node, "lineno", "None"),
'BroadcastGlobalVariablesHook')
node.func = ast.Name(id="NpuEmptyHook", ctx=ast.Load())
node.args = []
node.keywords = []
util_global.set_value('need_conver', True)
return node
if isinstance(node.func,
ast.Attribute) and node.func.attr == "DistributedOptimizer":
log_success_report(getattr(node, "lineno", "None"),
'DistributedOptimizer')
return node.args[0]
if isinstance(node.func, ast.Attribute) and node.func.attr == 'shard':
log_success_report(getattr(node, "lineno", "None"), 'shard')
node.args = [
ast.Call(func=ast.Name(id='get_rank_size', ctx=ast.Load()),
args=[],
keywords=[]),
ast.Call(func=ast.Name(id='get_rank_id', ctx=ast.Load()),
args=[],
keywords=[])
]
util_global.set_value("has_hccl_api", True)
util_global.set_value('need_conver', True)
if isinstance(node.func, ast.Attribute) and node.func.attr == 'dropout':
if isinstance(node.func.value,
ast.Attribute) and node.func.value.attr == 'nn':
log_success_report(getattr(node, "lineno", "None"), 'dropout')
node.func = ast.Attribute(value=ast.Name(id='npu_ops',
ctx=ast.Load()),
attr='dropout',
ctx=ast.Load())
keywords_new = []
for keyword in node.keywords:
if keyword.arg != 'rate':
keywords_new.append(keyword)
node.keywords = keywords_new
util_global.set_value('need_conver', True)
if isinstance(node.func, ast.Attribute) and ((node.func.attr == 'map_and_batch') or (node.func.attr == 'batch' \
and (not isinstance(node.func.value, ast.Attribute) or (isinstance(node.func.value, ast.Attribute) and node.func.value.attr != 'train')))):
exist = False
for keyword in node.keywords:
if keyword.arg == 'drop_remainder':
exist = True
if ((isinstance(keyword.value, ast.NameConstant)
and keyword.value.value != True)
or (not isinstance(keyword.value, ast.NameConstant))):
log_success_report(getattr(node, "lineno", "None"),
node.func.attr)
keyword.value = pasta.parse('True')
util_global.set_value('need_conver', True)
if not exist:
log_success_report(getattr(node, "lineno", "None"), node.func.attr)
keyword = ast.keyword(arg='drop_remainder',
value=pasta.parse('True'))
node.keywords.insert(0, keyword)
util_global.set_value('need_conver', True)
if (isinstance(node.func, ast.Attribute)
and isinstance(node.func.value, ast.Name)
and node.func.value.id == 'tf' and node.func.attr == 'device'):
log_success_report(getattr(node, "lineno", "None"), node.func.attr)
node.args = [ast.Str(s='/cpu:0')]
util_global.set_value('need_conver', True)
if isinstance(node.func, ast.Attribute) and (
node.func.attr == "get_distribution_strategy"
or node.func.attr == "MirroredStrategy"
or node.func.attr == "MultiWorkerMirroredStrategy"):
log_success_report(getattr(node, "lineno", "None"), node.func.attr)
new_func = ast.Attribute(value=ast.Name(id="npu_strategy",
ctx=ast.Load()),
attr="NPUStrategy",
ctx=ast.Load())
ast.copy_location(new_func, node.func)
node.func = new_func
node.keywords = []
node.args = []
util_global.set_value('need_conver', True)
if isinstance(node.func, ast.Attribute) and (node.func.attr
== 'AdviceProto'):
log_success_report(getattr(node, 'lineno', 'None'),
'tf.profiler.AdviceProto')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr == 'Checker'):
log_success_report(getattr(node, 'lineno', 'None'),
'tf.profiler.AdviceProto.Checker')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr
== 'CheckersEntry'):
log_success_report(getattr(node, 'lineno', 'None'),
'tf.profiler.AdviceProto.CheckersEntry')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr
== 'GraphNodeProto'):
log_success_report(getattr(node, 'lineno', 'None'),
'tf.profiler.GraphNodeProto')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr
== 'InputShapesEntry'):
log_success_report(getattr(node, 'lineno', 'None'),
'tf.profiler.GraphNodeProto.InputShapesEntry')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr
== 'MultiGraphNodeProto'):
log_success_report(getattr(node, 'lineno', 'None'),
'tf.profiler.MultiGraphNodeProto')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr
== 'OpLogProto'):
log_success_report(getattr(node, 'lineno', 'None'),
'tf.profiler.OpLogProto')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr
== 'IdToStringEntry'):
log_success_report(getattr(node, 'lineno', 'None'),
'tf.profiler.OpLogProto.IdToStringEntry')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr
== 'ProfileOptionBuilder'):
log_success_report(getattr(node, 'lineno', 'None'),
'tf.profiler.ProfileOptionBuilder')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr == 'advise'):
log_success_report(getattr(node, 'lineno', 'None'),
'tf.profiler.advise')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr == 'profile'):
log_success_report(getattr(node, 'lineno', 'None'),
'tf.profiler.profile')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr
== 'write_op_log'):
log_success_report(getattr(node, 'lineno', 'None'),
'tf.profiler.write_op_log')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr == 'TPUEstimator') and \
((isinstance(node.func.value, ast.Attribute) and (node.func.value.attr == 'tpu')) or \
(isinstance(node.func.value, ast.Name) and (node.func.value.id == 'tpu'))):
add_eval_on_tpu = True
add_use_tpu = True
add_export_to_tpu = True
for keyword in node.keywords:
if (keyword.arg == 'eval_on_tpu') or (
keyword.arg == 'use_tpu') or (keyword.arg
== 'export_to_tpu'):
if (not isinstance(keyword.value, ast.NameConstant)) or (
isinstance(keyword.value, ast.NameConstant) and
(keyword.value.value != False)):
log_success_report(getattr(node, 'lineno', 'None'),
'TPUEstimator(' + keyword.arg + '=*)')
keyword.value = pasta.parse('False')
util_global.set_value('need_conver', True)
if add_eval_on_tpu and (keyword.arg == 'eval_on_tpu'):
add_eval_on_tpu = False
if add_use_tpu and (keyword.arg == 'use_tpu'):
add_use_tpu = False
if add_export_to_tpu and (keyword.arg == 'export_to_tpu'):
add_export_to_tpu = False
if add_eval_on_tpu:
log_success_report(getattr(node, 'lineno', 'None'),
'TPUEstimator(eval_on_tpu=*)')
node.keywords.append(
ast.keyword(arg='eval_on_tpu', value=pasta.parse('False')))
util_global.set_value('need_conver', True)
if add_use_tpu:
log_success_report(getattr(node, 'lineno', 'None'),
'TPUEstimator(use_tpu=*)')
node.keywords.append(
ast.keyword(arg='use_tpu', value=pasta.parse('False')))
util_global.set_value('need_conver', True)
if add_export_to_tpu:
log_success_report(getattr(node, 'lineno', 'None'),
'TPUEstimator(export_to_tpu=*)')
node.keywords.append(
ast.keyword(arg='export_to_tpu', value=pasta.parse('False')))
util_global.set_value('need_conver', True)
if isinstance(node.func,
ast.Attribute) and (node.func.attr
== 'VirtualDeviceConfiguration'):
log_success_report(getattr(node, 'lineno', 'None'),
'VirtualDeviceConfiguration')
util_global.set_value('need_conver', True)
memory_limit = None
for keyword in node.keywords:
if keyword.arg == 'memory_limit':
memory_limit = keyword
break
if memory_limit:
memory_limit.value = ast.NameConstant(value=None)
else:
node.keywords.append(
ast.keyword(arg='memory_limit',
value=ast.NameConstant(value=None)))
return node
if isinstance(node.func,
ast.Attribute) and (node.func.attr
== 'set_soft_device_placement'):
log_success_report(getattr(node, 'lineno', 'None'),
'set_soft_device_placement')
util_global.set_value('need_conver', True)
node.args = []
node.keywords = [
ast.keyword(arg='enabled', value=ast.NameConstant(value=True))
]
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr
== 'set_memory_growth'):
log_success_report(getattr(node, 'lineno', 'None'),
'set_memory_growth')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func,
ast.Attribute) and (node.func.attr
== 'set_virtual_device_configuration'):
log_success_report(getattr(node, 'lineno', 'None'),
'set_virtual_device_configuration')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr
== 'jit_scope'):
if isinstance(node.func.value, ast.Attribute) and (node.func.value.attr
== 'experimental'):
if isinstance(node.func.value.value,
ast.Attribute) and (node.func.value.value.attr
== 'xla'):
log_success_report(getattr(node, 'lineno', 'None'),
'*.xla.experimental.jit_scope')
util_global.set_value('need_conver', True)
compile_ops = None
for keyword in node.keywords:
if keyword.arg == 'compile_ops':
compile_ops = keyword
break
if compile_ops:
compile_ops.value = pasta.parse('False')
else:
node.keywords.append(
ast.keyword(arg='compile_ops',
value=pasta.parse('False')))
return node
for estimator in util_global.get_value('Estimators', []):
if (isinstance(node.func, ast.Attribute) and (node.func.attr == estimator)) \
or (isinstance(node.func, ast.Name) and (node.func.id == estimator)):
log_msg(getattr(node, 'lineno'),
estimator + '() add config=npu_run_config_init()')
config = None
for keyword in node.keywords:
if keyword.arg == 'config':
config = keyword
break
if config:
new_value = ast.Call(func=ast.Name(id='npu_run_config_init',
ctx=ast.Load()),
args=[],
keywords=[
ast.keyword(arg='run_config',
value=config.value)
])
ast.copy_location(new_value, config.value)
config.value = new_value
else:
node.keywords.append(
ast.keyword(arg='config',
value=pasta.parse('npu_run_config_init()')))
util_global.set_value('need_conver', True)
return node
for estimator_func in util_global.get_value('EstimatorFunc', []):
if isinstance(node.func, ast.Attribute) and (node.func.attr
== estimator_func):
if isinstance(
node.func.value,
ast.Attribute) and node.func.value.attr == "learning":
return node
input_fn = None
hooks = None
for index, _ in enumerate(node.args):
if index == 0:
input_fn = node.args[0]
elif index == 1:
hooks = node.args.pop(1)
for keyword in node.keywords:
if keyword.arg == 'input_fn':
input_fn = keyword
elif keyword.arg == 'hooks':
hooks = keyword
if not input_fn:
break
if not hooks:
node.keywords.append(
ast.keyword(arg='hooks',
value=pasta.parse('npu_hooks_append()')))
elif isinstance(hooks, ast.keyword):
new_value = ast.Call(func=ast.Name(id='npu_hooks_append',
ctx=ast.Load()),
args=[],
keywords=[
ast.keyword(arg='hooks_list',
value=hooks.value)
])
ast.copy_location(new_value, hooks.value)
hooks.value = new_value
else:
node.keywords.append(
ast.keyword(arg='hooks',
value=ast.Call(
func=ast.Name(id='npu_hooks_append',
ctx=ast.Load()),
args=[],
keywords=[
ast.keyword(arg='hooks_list',
value=hooks)
])))
util_global.set_value('need_conver', True)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr == 'compile'):
opt_map = {
"adadelta": "tf.keras.optimizers.Adadelta()",
"adagrad": "tf.keras.optimizers.Adagrad()",
"adam": "tf.keras.optimizers.Adam()",
"adamax": "tf.keras.optimizers.Adamax()",
"ftrl": "tf.keras.optimizers.Ftrl()",
"nadam": "tf.keras.optimizers.Nadam()",
"rmsprop": "tf.keras.optimizers.RMSprop()",
"sgd": "tf.keras.optimizers.SGD()"
}
for keyword in node.keywords:
if keyword.arg == "optimizer":
log_success_report(getattr(node, 'lineno', 'None'),
'KerasDistributeOptimizer')
if isinstance(keyword.value, ast.Str):
keras_opt = opt_map[keyword.value.s]
npu_keras_opt = "npu_keras_optimizer(" + keras_opt + ")"
keyword.value = pasta.parse(npu_keras_opt)
util_global.set_value('need_conver', True)
return node
if isinstance(node.func, ast.Attribute) and isinstance(
node.func.value, ast.Attribute):
if (node.func.attr.find("Optimizer") !=
-1) and (node.func.attr != 'ScipyOptimizerInterface'):
log_msg(getattr(node, "lineno", "None"),
"add NPUDistributedOptimizer()")
new_node = ast.Call(func=ast.Name(id="npu_tf_optimizer",
ctx=ast.Load()),
args=[node],
keywords=[])
ast.copy_location(new_node, node)
util_global.set_value('need_conver', True)
return new_node
if isinstance(node.func, ast.Attribute):
opt_list = [
"Adadelta", "Adagrad", "Adam", "Adamax", "Ftrl", "Nadam",
"RMSprop", "SGD"
]
if node.func.attr in opt_list:
log_success_report(getattr(node, "lineno", "None"),
"KerasDistributeOptimizer")
new_node = ast.Call(func=ast.Name(id="npu_keras_optimizer",
ctx=ast.Load()),
args=[node],
keywords=[])
ast.copy_location(new_node, node)
util_global.set_value('need_conver', True)
return new_node
if (isinstance(node.func, ast.Attribute) and (node.func.attr == 'MonitoredTrainingSession')) or \
(isinstance(node.func, ast.Name) and (node.func.id == 'MonitoredTrainingSession')):
log_success_report(getattr(node, "lineno", "None"),
'MonitoredTrainingSession')
hooks = None
for index, _ in enumerate(node.args):
if index == 4:
hooks = node.args.pop(4)
break
for keyword in node.keywords:
if keyword.arg == 'hooks':
hooks = keyword
break
if not hooks:
node.keywords.append(
ast.keyword(arg='hooks',
value=pasta.parse('npu_hooks_append()')))
elif isinstance(hooks, ast.keyword):
new_value = ast.Call(
func=ast.Name(id='npu_hooks_append', ctx=ast.Load()),
args=[],
keywords=[ast.keyword(arg='hooks_list', value=hooks.value)])
ast.copy_location(new_value, hooks.value)
hooks.value = new_value
else:
node.keywords.append(
ast.keyword(arg='hooks',
value=ast.Call(func=ast.Name(id='npu_hooks_append',
ctx=ast.Load()),
args=[],
keywords=[
ast.keyword(arg='hooks_list',
value=hooks)
])))
util_global.set_value('need_conver', True)
return node
specs = {'TrainSpec': 2, 'EvalSpec': 3}
for spec, hooks_index in specs.items():
if _call_name_match(node.func, spec):
log_success_report(getattr(node, "lineno", "None"), spec)
hooks = None
for index, _ in enumerate(node.args):
if index == hooks_index:
hooks = node.args.pop(hooks_index)
break
for keyword in node.keywords:
if keyword.arg == 'hooks':
hooks = keyword
break
if not hooks:
node.keywords.append(
ast.keyword(arg='hooks',
value=pasta.parse('npu_hooks_append()')))
elif isinstance(hooks, ast.keyword):
new_value = ast.Call(func=ast.Name(id='npu_hooks_append',
ctx=ast.Load()),
args=[],
keywords=[
ast.keyword(arg='hooks_list',
value=hooks.value)
])
ast.copy_location(new_value, hooks.value)
hooks.value = new_value
else:
node.keywords.append(
ast.keyword(arg='hooks',
value=ast.Call(
func=ast.Name(id='npu_hooks_append',
ctx=ast.Load()),
args=[],
keywords=[
ast.keyword(arg='hooks_list',
value=hooks)
])))
util_global.set_value('need_conver', True)
return node
def sample_gibbs(self,
n_nodes,
n_samples=1,
burn_in=None,
n_steps=None,
g0=None,
print_logs=None):
"""
Sample from this ensemble, returning a 3d numpy array which is `n_nodes x n_nodes x n_samples` and a 2d
numpy array which is `d x n_samples`, where `d` is the number of statistics. The second array stores the
statistics of each sample, to avoid recomputing them (e.g. in parameter estimation)
:param n_nodes: Number of nodes in the graph
:param n_samples: Number of samples to return
:param burn_in: Number of burn-in steps
:param n_steps: Number of steps between samples
:param g0: Initial adjacency matrix to use. Default is previous internal state for sampler, if appropriate
:param print_logs: where to print logs. Default is None, suppressing output
:return: A numpy array of integers (the adjacency matrices) and a numpy array of floats (the statistics)
This method uses Gibbs sampling.
"""
# TODO write up some details on the internals of this method in the docstring
# Compute number of steps this MC will take
if burn_in is None:
if self.current_adj.shape[0] == n_nodes:
burn_in = 0
else:
burn_in = 3 * int(math.ceil(
n_nodes * math.log(n_nodes))) * len(self.theta)
# above is based on some rough estimates/simulations
if n_steps is None:
n_steps = int(math.ceil(math.log(n_nodes))) * len(self.theta) * 2
total_steps = burn_in + n_steps * n_samples # total steps taken by markov chain
# initialize output arrays
if self.use_sparse:
samples = np.array([
sparse.csr_matrix((n_nodes, n_nodes), dtype=int)
for _ in range(n_samples)
])
else:
samples = np.zeros((n_nodes, n_nodes, n_samples), dtype=int)
sample_stats = np.zeros((self.theta.shape[0], n_samples))
# determine the sequence of edges to visit
urand = np.random.rand(total_steps)
# rand_indexes = np.random.choice(range(n_nodes * (n_nodes - 1) // (1 + (not self.directed))), size=total_steps, replace=True)
rand_indexes = np.random.randint(0,
n_nodes * (n_nodes - 1) //
(1 + (not self.directed)),
size=total_steps)
edge_sequence = index_to_edge(rand_indexes, n_nodes, self.directed)
# TODO There might be a way to speed up indexing into a CSR matrix by reformatting the edge sequence
if g0 is None and self.current_adj.shape[0] == n_nodes:
log_msg("sample_gibbs: previous adjacency matrix found",
out=print_logs)
# burn_in = 0 # we're picking up where we left off
# pass
elif g0 is None:
log_msg("sample_gibbs: using empty graph for initial state",
out=print_logs)
# self.current_adj = np.zeros((n_nodes, n_nodes))
# self.current_stats = self.stats(self.current_adj)
# self.current_logweight = np.dot(self.current_stats, self.theta)
# TODO enable random initialization
if self.use_sparse:
self._initialize_sparse_adj(n_nodes,
edge_sequence,
reset_stats=True)
else:
self._initialize_dense_adj(n_nodes, reset_stats=True)
self.proposed_stats = np.zeros_like(self.current_stats)
# self.current_logweight = self.logweight(self.current_adj)
else:
log_msg(
"sample_gibbs: using provided adjacency matrix for initial state",
out=print_logs)
self.current_adj = g0
self.current_stats = self.stats(self.current_adj)
self.current_logweight = self.theta.dot(self.current_stats)
self.proposed_stats = np.zeros_like(self.current_stats)
# self.current_logweight = self.logweight(g0)
log_msg("sample_gibbs: %8d nodes" % n_nodes, out=print_logs)
log_msg("sample_gibbs: %8d burn-in steps" % burn_in, out=print_logs)
log_msg("sample_gibbs: %8d steps between samples" % n_steps,
out=print_logs)
log_msg("sample_gibbs: %8d total steps" % total_steps, out=print_logs)
log_msg("sample_gibbs: beginning MCMC process", out=print_logs)
t_start = time.time()
for step in range(total_steps):
# assuming the logweight of the current state is already computed, we just need to compute the new values
delta_k = self.delta_stats(self.current_adj, edge_sequence[0,
step],
edge_sequence[1, step])
self.proposed_stats[:] = self.current_stats[:] - delta_k[:] # the [:] are there to avoid new allocations(?)
self.proposed_logweight = self.theta.dot(self.proposed_stats)
p_flip = 1 / (1 + math.exp(self.theta.dot(delta_k)))
if urand[step] < p_flip:
# flip the edge, save the logweight and stats
self._toggle_current_edge(edge_sequence[0, step],
edge_sequence[1, step])
self.current_stats[:] = self.proposed_stats[:]
self.current_logweight = self.proposed_logweight
# avoid modifying self.current_adj, which may be sparse, until we're sure we're flipping the edge.
if step >= burn_in and (step - burn_in) % n_steps == 0:
# emit sample
sample_num = (step - burn_in) // n_steps
if print_logs is not None and (sample_num + 1) % (n_samples //
10) == 0:
log_msg("sample_gibbs: emitting sample %8d / %8d" %
(sample_num + 1, n_samples),
out=print_logs)
# samples[:, :, sample_num] = self.current_adj[:, :]
if self.use_sparse:
# samples[sample_num][:, :] = self.current_adj[:, :]
samples[sample_num] = self.current_adj.copy()
else:
samples[:, :, sample_num] = self.current_adj[:, :]
sample_stats[:, sample_num] = self.current_stats[:]
t_end = time.time()
log_msg("sample_gibbs: Sampling finished in",
t_end - t_start,
"s",
out=print_logs)
return samples, sample_stats
def ast_call(node):
"""Visit and transform ast call node"""
distributed_mode = util_global.get_value("distributed_mode", "")
is_not_strategy = distributed_mode in ("horovod", "")
is_not_horovod = distributed_mode in ("tf_strategy", "")
convert_loss_scale_api(node)
if _call_name_match(node.func, "set_experimental_options"):
log_msg(
getattr(node, 'lineno', 'None'),
'change set_experimental_options(*) to set_experimental_options(experimental_options)'
)
node.args = [ast.Name(id='experimental_options', ctx=ast.Load())]
node.keywords = []
util_global.set_value('need_conver', True)
if isinstance(node.func,
ast.Name) and node.func.id == 'check_available_gpus':
log_msg(getattr(node, 'lineno', 'None'),
"change check_available_gpus() to ['/device:CPU:0']")
util_global.set_value('need_conver', True)
return ast.List(elts=[ast.Str(s="/device:CPU:0")], ctx=ast.Load())
if ((isinstance(node.func, ast.Name) and node.func.id == 'GraphOptions')
or (isinstance(node.func, ast.Attribute)
and node.func.attr == 'GraphOptions')):
log_success_report(getattr(node, 'lineno', 'None'), 'GraphOptions()')
src = copy.deepcopy(node)
node.func = ast.Name(id='npu_graph_options', ctx=ast.Load())
node.args = []
node.keywords = []
node.keywords.append(ast.keyword(arg='graph_options', value=src))
util_global.set_value('need_conver', True)
return node
if (isinstance(node.func, ast.Name) and node.func.id == 'OptimizerOptions') or \
(isinstance(node.func, ast.Attribute) and node.func.attr == 'OptimizerOptions'):
log_success_report(getattr(node, 'lineno', 'None'),
'OptimizerOptions()')
src = copy.deepcopy(node)
node.func = ast.Name(id='npu_optimizer_options', ctx=ast.Load())
node.args = []
node.keywords = []
node.keywords.append(ast.keyword(arg='optimizer_options', value=src))
util_global.set_value('need_conver', True)
return node
if _call_name_match(node.func, "Session"):
return convert_origin_func_to_npu(node, tf_func_map["tf.Session"],
"tf.Session", ["config"])
if _call_name_match(node.func, "InteractiveSession"):
return convert_origin_func_to_npu(node,
tf_func_map["tf.InteractiveSession"],
"tf.InteractiveSession", ["config"])
if isinstance(node.func, ast.Attribute
) and node.func.attr == "BroadcastGlobalVariablesHook":
if isinstance(node.func.value,
ast.Name) and node.func.value.id == "hvd":
if is_not_horovod:
log_strategy_distributed_mode_error(node)
return node
log_msg(
getattr(node, "lineno", "None"),
'change hvd.BroadcastGlobalVariablesHook to NPUBroadcastGlobalVariablesHook'
)
node = pasta.parse(
"NPUBroadcastGlobalVariablesHook(0, int(os.getenv('RANK_ID', '0')))"
)
util_global.set_value('need_conver', True)
return node
if isinstance(node.func, ast.Attribute
) and node.func.attr == "BroadcastGlobalVariablesCallback":
if isinstance(node.func.value,
ast.Attribute) and node.func.value.attr == "callbacks":
if is_not_horovod:
log_strategy_distributed_mode_error(node)
return node
log_msg(
getattr(node, "lineno", "None"),
'change hvd.callbacks.BroadcastGlobalVariablesCallback to NPUBroadcastGlobalVariablesCallback'
)
node = pasta.parse(
"NPUBroadcastGlobalVariablesCallback(root_rank=0)")
util_global.set_value('need_conver', True)
return node
if isinstance(node.func,
ast.Attribute) and node.func.attr == "DistributedOptimizer":
if isinstance(node.func.value,
ast.Name) and node.func.value.id == "hvd":
if is_not_horovod:
log_strategy_distributed_mode_error(node)
return node
return convert_hvd_distributed_api(node)
if isinstance(node.func, ast.Attribute) and node.func.attr == 'shard':
log_success_report(getattr(node, "lineno", "None"), 'shard')
node.args = [
pasta.parse("int(os.getenv('RANK_SIZE', '1'))"),
pasta.parse("int(os.getenv('RANK_ID', '0'))")
]
node.keywords.clear()
util_global.set_value('need_conver', True)
return node
if isinstance(node.func, ast.Attribute) and node.func.attr == 'dropout':
if isinstance(node.func.value,
ast.Attribute) and node.func.value.attr == 'nn':
for index, _ in enumerate(node.args):
if index == 2:
return node
for keyword in node.keywords:
if keyword.arg == "noise_shape":
return node
log_success_report(getattr(node, "lineno", "None"), 'dropout')
node.func = ast.Attribute(value=ast.Name(id='npu_ops',
ctx=ast.Load()),
attr='dropout',
ctx=ast.Load())
keywords_new = []
for keyword in node.keywords:
if keyword.arg != 'rate':
keywords_new.append(keyword)
else:
keywords_new.append(
ast.keyword(arg='keep_prob',
value=ast.BinOp(left=ast.Num(n=1),
op=ast.Sub(),
right=keyword.value)))
node.keywords = keywords_new
util_global.set_value('need_conver', True)
return node
if isinstance(node.func, ast.Attribute) and \
((node.func.attr == 'map_and_batch') or
(node.func.attr == 'batch' and (not isinstance(node.func.value, ast.Attribute) or (
isinstance(node.func.value, ast.Attribute) and node.func.value.attr != 'train')))):
exist = False
for keyword in node.keywords:
if keyword.arg == 'drop_remainder':
exist = True
if ((isinstance(keyword.value, ast.NameConstant)
and not keyword.value.value)
or (not isinstance(keyword.value, ast.NameConstant))):
log_success_report(getattr(node, "lineno", "None"),
node.func.attr)
keyword.value = pasta.parse('True')
util_global.set_value('need_conver', True)
if not exist:
log_success_report(getattr(node, "lineno", "None"), node.func.attr)
keyword = ast.keyword(arg='drop_remainder',
value=pasta.parse('True'))
node.keywords.insert(0, keyword)
util_global.set_value('need_conver', True)
return node
if (isinstance(node.func, ast.Attribute)
and isinstance(node.func.value, ast.Name)
and node.func.value.id == 'tf' and node.func.attr == 'device'):
log_success_report(getattr(node, "lineno", "None"), node.func.attr)
node.args = [ast.Str(s='/cpu:0')]
util_global.set_value('need_conver', True)
return node
if isinstance(node.func, ast.Attribute) and \
(node.func.attr == "get_distribution_strategy" or
node.func.attr == "MirroredStrategy" or
node.func.attr == "MultiWorkerMirroredStrategy"):
if is_not_strategy:
log_hvd_distributed_mode_error(node)
return node
log_success_report(getattr(node, "lineno", "None"), node.func.attr)
new_func = ast.Attribute(value=ast.Name(id="npu_strategy",
ctx=ast.Load()),
attr="NPUStrategy",
ctx=ast.Load())
ast.copy_location(new_func, node.func)
node.func = new_func
node.keywords = []
node.args = []
util_global.set_value('need_conver', True)
return node
if (isinstance(node.func, ast.Attribute) and (node.func.attr == 'RunConfig')) and \
(_call_name_match(node.func.value, 'estimator') or _call_name_match(node.func.value, 'tpu')):
if node.keywords.count("train_distribute") or node.keywords.count(
"eval_distribute"):
if is_not_strategy:
log_hvd_distributed_mode_error(node)
save_summary_steps = None
for keyword in node.keywords:
if keyword.arg == 'save_summary_steps':
save_summary_steps = keyword
break
if len(node.args) < 3 and not save_summary_steps:
log_msg(getattr(node, 'lineno'),
'RunConfig() add save_summary_steps=0')
util_global.set_value('need_conver', True)
node.keywords.append(
ast.keyword(arg='save_summary_steps', value=pasta.parse('0')))
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr == 'TPUEstimator') and \
((isinstance(node.func.value, ast.Attribute) and (node.func.value.attr == 'tpu')) or
(isinstance(node.func.value, ast.Name) and (node.func.value.id == 'tpu'))):
add_eval_on_tpu = True
add_use_tpu = True
add_export_to_tpu = True
for keyword in node.keywords:
if (keyword.arg == 'eval_on_tpu') or (
keyword.arg == 'use_tpu') or (keyword.arg
== 'export_to_tpu'):
if (not isinstance(keyword.value, ast.NameConstant)) or \
(isinstance(keyword.value, ast.NameConstant) and (keyword.value.value)):
log_success_report(getattr(node, 'lineno', 'None'),
'TPUEstimator(' + keyword.arg + '=*)')
keyword.value = pasta.parse('False')
util_global.set_value('need_conver', True)
if add_eval_on_tpu and (keyword.arg == 'eval_on_tpu'):
add_eval_on_tpu = False
if add_use_tpu and (keyword.arg == 'use_tpu'):
add_use_tpu = False
if add_export_to_tpu and (keyword.arg == 'export_to_tpu'):
add_export_to_tpu = False
if add_eval_on_tpu:
log_success_report(getattr(node, 'lineno', 'None'),
'TPUEstimator(eval_on_tpu=*)')
node.keywords.append(
ast.keyword(arg='eval_on_tpu', value=pasta.parse('False')))
util_global.set_value('need_conver', True)
if add_use_tpu:
log_success_report(getattr(node, 'lineno', 'None'),
'TPUEstimator(use_tpu=*)')
node.keywords.append(
ast.keyword(arg='use_tpu', value=pasta.parse('False')))
util_global.set_value('need_conver', True)
if add_export_to_tpu:
log_success_report(getattr(node, 'lineno', 'None'),
'TPUEstimator(export_to_tpu=*)')
node.keywords.append(
ast.keyword(arg='export_to_tpu', value=pasta.parse('False')))
util_global.set_value('need_conver', True)
if isinstance(node.func,
ast.Attribute) and (node.func.attr
== 'VirtualDeviceConfiguration'):
log_success_report(getattr(node, 'lineno', 'None'),
'VirtualDeviceConfiguration')
util_global.set_value('need_conver', True)
memory_limit = None
for keyword in node.keywords:
if keyword.arg == 'memory_limit':
memory_limit = keyword
break
if memory_limit:
memory_limit.value = ast.NameConstant(value=None)
else:
node.keywords.append(
ast.keyword(arg='memory_limit',
value=ast.NameConstant(value=None)))
return node
if isinstance(node.func,
ast.Attribute) and (node.func.attr
== 'set_soft_device_placement'):
log_success_report(getattr(node, 'lineno', 'None'),
'set_soft_device_placement')
util_global.set_value('need_conver', True)
node.args = []
node.keywords = [
ast.keyword(arg='enabled', value=ast.NameConstant(value=True))
]
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr
== 'set_memory_growth'):
log_success_report(getattr(node, 'lineno', 'None'),
'set_memory_growth')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func,
ast.Attribute) and (node.func.attr
== 'set_virtual_device_configuration'):
log_success_report(getattr(node, 'lineno', 'None'),
'set_virtual_device_configuration')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr
== 'jit_scope'):
if isinstance(node.func.value, ast.Attribute) and (node.func.value.attr
== 'experimental'):
if isinstance(node.func.value.value,
ast.Attribute) and (node.func.value.value.attr
== 'xla'):
log_success_report(getattr(node, 'lineno', 'None'),
'*.xla.experimental.jit_scope')
util_global.set_value('need_conver', True)
compile_ops = None
for keyword in node.keywords:
if keyword.arg == 'compile_ops':
compile_ops = keyword
break
if compile_ops:
compile_ops.value = pasta.parse('False')
else:
node.keywords.append(
ast.keyword(arg='compile_ops',
value=pasta.parse('False')))
return node
for estimator in util_global.get_value('Estimators', []):
if (isinstance(node.func, ast.Attribute) and (node.func.attr == estimator)) \
or (isinstance(node.func, ast.Name) and (node.func.id == estimator)):
log_msg(
getattr(node, 'lineno'),
"".join([estimator, '() add config=npu_run_config_init()']))
config = None
for keyword in node.keywords:
if keyword.arg == 'config':
config = keyword
break
if config:
new_value = ast.Call(func=ast.Name(id='npu_run_config_init',
ctx=ast.Load()),
args=[],
keywords=[
ast.keyword(arg='run_config',
value=config.value)
])
ast.copy_location(new_value, config.value)
config.value = new_value
else:
node.keywords.append(
ast.keyword(arg='config',
value=pasta.parse('npu_run_config_init()')))
util_global.set_value('need_conver', True)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr
== 'clear_session'):
log_msg(getattr(node, 'lineno'),
"change keras.clear_session() to npu_clear_session()")
node = ast.Call(func=ast.Name(id='npu_clear_session', ctx=ast.Load()),
args=[],
keywords=[])
util_global.set_value('need_conver', True)
if _call_name_match(node.func, "MonitoredTrainingSession"):
return convert_origin_func_to_npu(
node, tf_func_map["tf.train.MonitoredTrainingSession"],
"MonitoredTrainingSession", ["config", "hooks"])
if isinstance(node.func,
ast.Attribute) and node.func.attr == "managed_session":
return convert_origin_func_to_npu(
node, tf_func_map["tf.train.Supervisor.managed_session"],
"managed_session", ["config"], True)
if distributed_mode == "tf_strategy": # this cond should be placed at the end of the Call function.
return convert_distributed_strategy_apis(node)
return node
def get_FlyBrain_graph(hemibrain_version="v1.1",
directory="/notebooks/nx_graph_utils/hemibrain/exported-traced-adjacencies-",
nodes_file="traced-neurons.csv",
edges_file="traced-total-connections.csv",
clusters_dir="/notebooks/nx_graph_utils/hemibrain/und_flybrain_v1.1/"):
"""Load the flybrain graph from the specified files. Returns a Networkx graph with weighted edges and labeled
nodes. """
directory = directory + hemibrain_version
log_msg("Attempting to read edges from", os.path.join(directory, edges_file))
edge_data = open(os.path.join(directory, edges_file), "r")
next(edge_data, None) # skip the first line of the file, which is just a header row
t_start = time.time()
FlyBrain = nx.parse_edgelist(edge_data, delimiter=",", create_using=nx.DiGraph, nodetype=int,
data=[("n_synapses", int)])
t_end = time.time()
edge_data.close()
log_msg("Read in {} nodes and {} edges in {} s:".format(nx.number_of_nodes(FlyBrain), nx.number_of_edges(FlyBrain),
t_end - t_start))
print(nx.info(FlyBrain))
node_data = open(os.path.join(directory, nodes_file), 'r')
# next(node_data,None)
log_msg("Attempting to read cell type data from {}".format(os.path.join(directory, nodes_file)))
line_counter = 0
for line in node_data.readlines()[1:]:
l = line.split(",")
ct = l[1].strip()
inst = l[2].strip()
if len(ct) == 0:
ct = "None"
if len(inst) == 0:
inst = "None"
nx.set_node_attributes(FlyBrain, {int(l[0]): {"celltype": ct, "instance": inst}}) # in v1.1 it's switched
node_data.seek(0)
n_clusters = {}
cluster_hist = {}
for filename in os.listdir(clusters_dir):
log_msg("Reading file", filename)
if "unweighted" in filename:
cluster_name = "unweighted"
elif "lr" in filename:
cluster_name = "lr"
else:
try:
cluster_name = (filename.split(".")[0]).split("_")[2]
cluster_name = cluster_name[1:].replace("p", ".")
except:
log_msg("Error reading file", filename, ", skipping file")
continue
# lines_read, clust_histogram = read_cluster_ids(FlyBrain, filename, cluster_name)
cluster_info = open(os.path.join(clusters_dir, filename), "r")
for nl, cl in zip(node_data.readlines()[1:], cluster_info.readlines()):
n_id = int(nl.split(",")[0])
c_id = int(cl)
nx.set_node_attributes(FlyBrain, {n_id: {cluster_name: c_id}})
n_clusters[cluster_name] = clust_histogram.shape[0]
cluster_hist[cluster_name] = clust_histogram
node_data.seek(0)
log_msg(n_clusters)
node_data.close()
return FlyBrain, n_clusters, cluster_hist
"""Testing the simplest version of the parameter estimation, using gradient ascent on the loglikelihood (using sample
estimates of the ensemble mean) """
import sys
import time
import networkx as nx
import numpy as np
from ergm import ERGM
from util import log_msg
log_msg("BEGIN SCRIPT:", __file__)
log_msg("Testing parameter estimation using sample mean of ensemble average")
n_nodes = 25
n_observed = 1000
seed = 17289
np.random.seed(seed)
max_edges = n_nodes * (n_nodes - 1) // 2
max_triangles = n_nodes * (n_nodes - 1) * (n_nodes - 2) // 6
theta_true = np.array([1.0, -10000.0]) # be fairly dense, but avoid triangles (cycles all length > 4)
log_msg("Using ergm to produce", n_observed, "observations of low-triangle graphs")
log_msg("theta =", theta_true)
def edge_triangle(g):
return np.array([g.sum() / (2 * max_edges), np.trace(g @ g @ g) / (6 * max_triangles)])
