def AddContract(self, contract):
"""
Add a contract to the database.
Args:
contract(neo.SmartContract.Contract): a Contract instance.
"""
super(UserWallet, self).AddContract(contract)
try:
db_contract = Contract.get(ScriptHash=contract.ScriptHash.ToBytes())
db_contract.delete_instance()
except Exception as e:
logger.info("contract does not exist yet")
sh = bytes(contract.ScriptHash.ToArray())
address, created = Address.get_or_create(ScriptHash=sh)
address.IsWatchOnly = False
address.save()
db_contract = Contract.create(RawData=contract.ToArray(),
ScriptHash=contract.ScriptHash.ToBytes(),
PublicKeyHash=contract.PublicKeyHash.ToBytes(),
Address=address,
Account=self.__dbaccount)
logger.debug("Creating db contract %s " % db_contract)
db_contract.save()
def __init__(self):
try:
self._db = plyvel.DB(settings.debug_storage_leveldb_path, create_if_missing=True)
except Exception as e:
logger.info("DEBUG leveldb unavailable, you may already be running this process: %s " % e)
raise Exception('DEBUG Leveldb Unavailable %s ' % e)
def main():
# Setup the blockchain
blockchain = LevelDBBlockchain(settings.chain_leveldb_path)
Blockchain.RegisterBlockchain(blockchain)
dbloop = task.LoopingCall(Blockchain.Default().PersistBlocks)
dbloop.start(.1)
NodeLeader.Instance().Start()
# Disable smart contract events for external smart contracts
settings.set_log_smart_contract_events(False)
# Start a thread with custom code
d = threading.Thread(target=custom_background_code)
d.setDaemon(True) # daemonizing the thread will kill it when the main thread is quit
d.start()
# Hook up Klein API to Twisted reactor.
endpoint_description = "tcp:port=%s:interface=localhost" % API_PORT
# If you want to make this service externally available (not only at localhost),
# then remove the `interface=localhost` part:
# endpoint_description = "tcp:port=%s" % API_PORT
endpoint = endpoints.serverFromString(reactor, endpoint_description)
endpoint.listen(Site(app.resource()))
# Run all the things (blocking call)
logger.info("Everything setup and running. Waiting for events...")
reactor.run()
logger.info("Shutting down.")
def StepInto(self):
if self._InvocationStack.Count == 0:
self._VMState |= VMState.HALT
if self._VMState & VMState.HALT > 0 or self._VMState & VMState.FAULT > 0:
logger.info("stopping because vm state is %s " % self._VMState)
return
op = None
if self.CurrentContext.InstructionPointer >= len(self.CurrentContext.Script):
op = RET
else:
op = self.CurrentContext.OpReader.ReadByte(do_ord=False)
self.ops_processed += 1
try:
if self._is_write_log:
self.write_log("{} {}".format(self.ops_processed, ToName(op)))
self.ExecuteOp(op, self.CurrentContext)
except Exception as e:
error_msg = "COULD NOT EXECUTE OP (%s): %s %s %s" % (self.ops_processed, e, op, ToName(op))
self.write_log(error_msg)
if self._exit_on_error:
self._VMState |= VMState.FAULT
else:
logger.error(error_msg)
logger.exception(e)
def run_test(endpoint_model, metrics_groups, alert_groups, endpoint_expected, endpoint_threshold):
attempt = 0
keep_trying = True
incident_timestamp = datetime.now(timezone.utc).astimezone().isoformat()
original_endpoint_url = endpoint_model.url
while keep_trying:
if "##CUPCAKE_ATTEMPT##" in original_endpoint_url:
endpoint_model.url = original_endpoint_url.replace("##CUPCAKE_ATTEMPT##", str(attempt + 1))
result = test_endpoint(
endpoint=endpoint_model,
expected=endpoint_expected,
threshold=endpoint_threshold,
metrics_groups=metrics_groups
)
if not result["result"] and result["message"] == "TIMEOUT":
attempt = attempt + 1
if attempt <= 3:
logger.info("re-testing timed out endpoint ({}) (attempt {} failed)".format(endpoint_model.url, attempt))
keep_trying = True
continue
break
incident = Incident(
timestamp=incident_timestamp,
endpoint=endpoint_model,
result=result,
expected=endpoint_expected
)
handle_result(
incident=incident,
alert_groups=alert_groups
)
def RelayDirectly(self, inventory):
"""
Relay the inventory to the remote client.
Args:
inventory (neo.Network.Inventory):
Returns:
bool: True if relayed successfully. False otherwise.
"""
relayed = False
self.RelayCache[inventory.Hash.ToBytes()] = inventory
for peer in self.Peers:
relayed |= peer.Relay(inventory)
if len(self.Peers) == 0:
if type(BC.Default()) is TestLevelDBBlockchain:
# mock a true result for tests
return True
logger.info("no connected peers")
return relayed
def get_by_addr(self, request, address):
request.setHeader('Content-Type', 'application/json')
try:
notifications = self.notif.get_by_addr(address)
except Exception as e:
logger.info("Could not get notifications for address %s " % address)
return self.format_message("Could not get notifications for address %s because %s" % (address, e))
return self.format_notifications(request, notifications)
def __init__(self, path):
try:
self._db = plyvel.DB(path, create_if_missing=True)
logger.info("Created Notification DB At %s " % path)
except Exception as e:
logger.info("Notification leveldb unavailable, you may already be running this process: %s " % e)
raise Exception('Notification Leveldb Unavailable %s ' % e)
def get_by_contract(self, request, contract_hash):
request.setHeader('Content-Type', 'application/json')
try:
hash = UInt160.ParseString(contract_hash)
notifications = self.notif.get_by_contract(hash)
except Exception as e:
logger.info("Could not get notifications for contract %s " % contract_hash)
return self.format_message("Could not get notifications for contract hash %s because %s" % (contract_hash, e))
return self.format_notifications(request, notifications)
def main():
sched.every(10).minutes.do(heartbeat)
next_min = minute_adder(1).strftime('%H:%M')
logger.info("When the clock strikes %s, down the rabbit hole with you!" % next_min)
sched.every().day.at(next_min).do(the_queue)
sched.every().day.at(UTCRebootTime).do(reboot)
while True:
sched.run_pending()
time.sleep(1)
def custom_background_code():
""" Custom code run in a background thread. Prints the current block height.
This function is run in a daemonized thread, which means it can be instantly killed at any
moment, whenever the main thread quits. If you need more safety, don't use a daemonized
thread and handle exiting this thread in another way (eg. with signals and events).
"""
while True:
logger.info("Block %s / %s", str(Blockchain.Default().Height), str(Blockchain.Default().HeaderHeight))
sleep(15)
def get_file_or_s3(uri):
logger.info("getting file URI %s" % uri)
if uri.lower().startswith("s3://"):
s3 = boto3.resource("s3")
parse_result = urlparse(uri)
s3_object = s3.Object(parse_result.netloc, parse_result.path.lstrip("/"))
return s3_object.get()["Body"].read().decode("utf-8")
return open(uri).read()
def Invoke(self, method, engine):
if method not in self._dictionary.keys():
logger.info("method %s not found in ->" % method)
for k, v in self._dictionary.items():
logger.info("%s -> %s " % (k, v))
return False
func = self._dictionary[method]
# logger.info("[InteropService Method] %s " % func)
return func(engine)
def sc_notify(event):
logger.info("SmartContract Runtime.Notify event: %s", event)
# Make sure that the event payload list has at least one element.
if not len(event.event_payload):
return
# The event payload list has at least one element. As developer of the smart contract
# you should know what data-type is in the bytes, and how to decode it. In this example,
# it's just a string, so we decode it with utf-8:
logger.info("- payload part 1: %s", event.event_payload[0].decode("utf-8"))
def get_vmstats():
dn = sys.argv[1]
fn = os.path.join(dn, 'vmstat.log')
if not os.path.exists(fn):
raise Exception('%s does not exist' % fn)
logger.info('reading %s' % fn)
with open(fn, 'r') as f:
lines = f.readlines()
rows = []
VMSTAT_TIMEZONE = None
for line in lines:
cols = line.split()
if cols[0] == 'procs':
continue
if cols[0] == 'r':
VMSTAT_TIMEZONE = cols[-1]
continue
data = {
'vmstat_r': int(cols[0]),
'vmstat_b': int(cols[1]),
'vmstat_swpd': int(cols[2]),
'vmstat_free': int(cols[3]),
'vmstat_buff': int(cols[4]),
'vmstat_cache': int(cols[5]),
'vmstat_si': int(cols[6]),
'vmstat_so': int(cols[7]),
'vmstat_bi': int(cols[8]),
'vmstat_bo': int(cols[9]),
'vmstat_in': int(cols[10]),
'vmstat_cs': int(cols[11]),
'vmstat_us': int(cols[12]),
'vmstat_sy': int(cols[13]),
'vmstat_id': int(cols[14]),
'vmstat_wa': int(cols[15]),
'vmstat_st': int(cols[16]),
}
# make a timezone aware timestamp
ts = cols[17] + ' ' + cols[18]
tstmp = datetime.datetime.strptime(ts, '%Y-%m-%d %H:%M:%S')
try:
tz = getattr(pytz, VMSTAT_TIMEZONE.lower())
except AttributeError as e:
tz = pytz.timezone(VMSTAT_TIMEZONE.upper())
tzts = datetime.datetime(tstmp.year, tstmp.month, tstmp.day, tstmp.hour, tstmp.minute, tstmp.second, tzinfo=tz)
data['ts'] = tzts.timestamp()
rows.append(data)
return rows
def get_by_block(self, request, block):
request.setHeader('Content-Type', 'application/json')
try:
if int(block) > Blockchain.Default().Height:
return self.format_message("Higher than current block")
else:
notifications = self.notif.get_by_block(block)
except Exception as e:
logger.info("Could not get notifications for block %s %s" % (block, e))
return self.format_message("Could not get notifications for block %s because %s " % (block, e))
return self.format_notifications(request, notifications)
def load_config(path):
"""Return fully instantiated config tree."""
if path is None:
log.info("returning empty config object.")
return Munch()
path = Path(path)
config = yaml.safe_load(path.open())
config = pathify_config(config)
config = munchify(config)
return config
def on_smart_contract_event(self, sc_event: NotifyEvent):
"""
Listener for NotifyEvent
Args:
sc_event (NotifyEvent): event to check whether it should be persisted
"""
if not isinstance(sc_event, NotifyEvent):
logger.info("Not Notify Event instance")
return
if sc_event.ShouldPersist:
if sc_event.notify_type == NotifyType.TRANSFER or sc_event.notify_type == NotifyType.REFUND:
self._events_to_write.append(sc_event)
def Verify(self, mempool):
"""
Verify the transaction.
Args:
mempool:
Returns:
bool: True if verified. False otherwise.
"""
logger.info("Verifying transaction: %s " % self.Hash.ToBytes())
return Helper.VerifyScripts(self)
def instance():
"""
Singleton accessor for NotificationDB
Returns:
NotificationDB: The current instance of the NotificationDB
"""
if not NotificationDB.__instance:
if settings.NOTIFICATION_DB_PATH:
NotificationDB.__instance = NotificationDB(settings.notification_leveldb_path)
# logger.info("Created Notification DB At %s " % settings.NOTIFICATION_DB_PATH)
else:
logger.info("Notification DB Path not configured in settings")
return NotificationDB.__instance
def get_token(self, request, contract_hash):
request.setHeader('Content-Type', 'application/json')
try:
uint160 = UInt160.ParseString(contract_hash)
contract_event = self.notif.get_token(uint160)
if not contract_event:
return self.format_message("Could not find contract with hash %s" % contract_hash)
notifications = [contract_event]
except Exception as e:
logger.info("Could not get contract with hash %s because %s " % (contract_hash, e))
return self.format_message("Could not get contract with hash %s because %s " % (contract_hash, e))
return self.format_notifications(request, notifications)
def validate_control_socket(SSHCMD):
# $ ssh -O check -o ControlPath=... vagrant@el6host
# Master running (pid=24779)
for idx, x in enumerate(SSHCMD):
if x.startswith('ControlPath'):
cppath = x.split('=')[1]
if not os.path.exists(cppath):
logger.info('%s does not exist' % cppath)
else:
cpcmd = SSHCMD[:-1]
checkcmd = cpcmd[:]
checkcmd.insert(-1, '-O')
checkcmd.insert(-1, 'check')
print('# %s' % ' '.join(checkcmd))
(rc, so, se) = run_ssh_cmd(
' '.join(checkcmd),
use_selectors=False
)
logger.debug('rc: %s' % rc)
logger.debug('so: %s' % so)
logger.debug('se: %s' % se)
if rc != 0 or so.strip():
logger.info('checkcmd rc != 0 or has stdout')
logger.info(so)
logger.info(se)
def install_conda_env(ctx):
"""Installs virtual environment."""
try:
with ctx.prefix(SOURCE_CONDA):
log.info("install conda environment")
ctx.run(
f"conda create -n {CONDA_ENV_NAME} 'python >=3.6' --file requirements.txt --yes"
)
except UnexpectedExit as err:
result = err.args[0]
if "prefix already exists" in result.stderr:
log.info("Conda env already exists; moving to next step.")
else:
log.error(err)
raise err
def get_observations_from_baseline():
dn = sys.argv[1]
fn = os.path.join(dn, 'baseline.json')
if not os.path.exists(fn):
raise Exception('%s does not exist' % fn)
logger.info('reading %s' % fn)
with open(fn, 'r') as f:
baseline = json.loads(f.read())
# adapt the baseline dict to events
observations = []
obs = {
'host': None,
'ts': baseline[0]['play']['duration']['start'],
'task_name': None,
#'fn': fn
}
observations.append(obs)
for task in baseline[0]['tasks']:
tn = task['task']['name']
for hn,hd in task['hosts'].items():
for key in ['start', 'end']:
obs = {
'task_name': tn,
'host': hn,
'ts': hd['duration'][key],
#'fn': fn
}
observations.append(obs)
#import epdb; epdb.st()
obs = {
'host': None,
'ts': baseline[0]['play']['duration']['end'],
'task_name': None,
#'fn': fn
}
observations.append(obs)
for idx,x in enumerate(observations):
ts = x['ts']
# 2019-02-22T02:22:21.249551
ts = datetime.datetime.strptime(ts, '%Y-%m-%dT%H:%M:%S.%f')
#ts = time.mktime(ts.timetuple())
ts = ts.timestamp()
observations[idx]['ts'] = ts
observations = sorted(observations, key=lambda x: x['ts'])
#import epdb; epdb.st()
return observations
def main():
logger.info("starting...")
setup_signal_handling()
global db
db = settings.get_database()
while lifecycle_continues():
lifecycle()
if lifecycle_continues():
logger.info("sleeping for %s seconds..." % settings.SLEEP_SECONDS)
for _ in range(settings.SLEEP_SECONDS):
if lifecycle_continues():
time.sleep(1)
def BuildAndRun(arguments, wallet, verbose=True, min_fee=DEFAULT_MIN_FEE, invocation_test_mode=True):
arguments, from_addr = get_from_addr(arguments)
path = get_arg(arguments)
contract_script = Compiler.instance().load_and_save(path)
newpath = path.replace('.py', '.avm')
logger.info("Saved output to %s " % newpath)
debug_map_path = path.replace('.py', '.debug.json')
debug_map = None
if os.path.exists(debug_map_path):
with open(debug_map_path, 'r') as dbg:
debug_map = json.load(dbg)
return DoRun(contract_script, arguments, wallet, path, verbose, from_addr, min_fee, invocation_test_mode, debug_map=debug_map)
def get_by_tx(self, request, tx_hash):
request.setHeader('Content-Type', 'application/json')
bc = Blockchain.Default() # type: Blockchain
notifications = []
try:
hash = UInt256.ParseString(tx_hash)
tx, height = bc.GetTransaction(hash)
block_notifications = self.notif.get_by_block(height - 1)
for n in block_notifications:
if n.tx_hash == tx.Hash:
notifications.append(n)
except Exception as e:
logger.info("Could not get tx with hash %s because %s " % (tx_hash, e))
return self.format_message("Could not get tx with hash %s because %s " % (tx_hash, e))
return self.format_notifications(request, notifications)
def get_observations_from_delphiki():
#fn = 'jobresults.2.7.7/.cache/observations.json'
#fn = 'rhtestOct17/270/.cache/observations.json'
#fn = 'rhtestOct17/242/.cache/observations.json'
#fn = 'rhtest-02-14-2019/jobresults.2.4.6.0/.cache/observations.json'
#fn = 'rhtest-02-14-2019/jobresults.2.8.0.dev0/.cache/observations.json'
dn = sys.argv[1]
fn = os.path.join(dn, '.cache', 'observations.json')
if not os.path.exists(fn):
raise Exception('%s does not exist' % fn)
logger.info('reading %s' % fn)
with open(fn, 'r') as f:
obs = json.loads(f.read())
logger.info('%s observations found' % len(obs))
return obs
def ToJson(self):
jsn = {}
if self.Script is not None:
if type(self.Script) is str:
jsn['script'] = self.Script
else:
jsn['script'] = self.Script.decode()
jsn['parameters'] = [p.ToJson() for p in self.ContractParameters]
if self.Signatures is not None:
jsn['signatures'] = {}
for key, value in self.Signatures.items():
if value is not None:
if type(value) is str:
jsn['signatures'][key] = value
else:
jsn['signatures'][key] = value.decode()
else:
logger.info("Seems like {} has empty signature".format(key))
return jsn
def main():
# Setup the blockchain
blockchain = LevelDBBlockchain(settings.chain_leveldb_path)
Blockchain.RegisterBlockchain(blockchain)
dbloop = task.LoopingCall(Blockchain.Default().PersistBlocks)
dbloop.start(.1)
NodeLeader.Instance().Start()
# Disable smart contract events for external smart contracts
settings.set_log_smart_contract_events(False)
# Start a thread with custom code
d = threading.Thread(target=custom_background_code)
d.setDaemon(True) # daemonizing the thread will kill it when the main thread is quit
d.start()
# Run all the things (blocking call)
logger.info("Everything setup and running. Waiting for events...")
reactor.run()
logger.info("Shutting down.")
def train(self, train_loader: DataLoader, valid_loader: DataLoader, opt_params: Optional[Mapping] = None,
nb_epoch=100, step=100, k=5, early=100, verbose=True, swa_warmup=None, **kwargs):
self.get_optimizer(**({} if opt_params is None else opt_params))
global_step, best_n5, e = 0, 0.0, 0
print_loss = 0.0 #
epoch_loss = np.zeros((1, nb_epoch), dtype=np.float)
np_p1 = np.zeros((1, nb_epoch), dtype=np.float)
np_p5 = np.zeros((1, nb_epoch), dtype=np.float)
for epoch_idx in range(nb_epoch):
print_epoch_loss = 0.0
if epoch_idx == swa_warmup:
self.swa_init()
p1_tmp = 0.0
p5_tmp = 0.0
for i, (train_x, train_y) in enumerate(train_loader, 1):
global_step += 1
loss = self.train_step(train_x, train_y.cuda())
print_loss += loss
print_epoch_loss += loss
if global_step % step == 0:
self.swa_step()
self.swap_swa_params()
##
labels = []
valid_loss = 0.0
self.model.eval()
with torch.no_grad():
for (valid_x, valid_y) in valid_loader:
# 原始寫法
# logits = self.model(valid_x)
logits = self.model(valid_x.long())
valid_loss += self.loss_fn(logits, valid_y.cuda()).item()
scores, tmp = torch.topk(logits, k)
labels.append(tmp.cpu())
valid_loss /= len(valid_loader)
labels = np.concatenate(labels)
##
# labels = np.concatenate([self.predict_step(valid_x, k)[1] for valid_x in valid_loader])
targets = valid_loader.dataset.data_y
p5, n5 = get_p_5(labels, targets), get_n_5(labels, targets)
p1, n1 = get_p_1(labels, targets), get_n_1(labels, targets)
p5_tmp = p5
p1_tmp = p1
if n5 > best_n5:
self.save_model(True) # epoch_idx > 1 * swa_warmup)
best_n5, e = n5, 0
else:
e += 1
if early is not None and e > early:
return
self.swap_swa_params()
if verbose:
log_msg = '%d %d train loss: %.7f valid loss: %.7f P@5: %.5f N@5: %.5f P@1: %.5f N@1: %.5f early stop: %d' % \
(epoch_idx, i * train_loader.batch_size, print_loss / step, valid_loss, round(p5, 5),
round(n5, 5), round(p1, 5), round(n1, 5), e)
logger.info(log_msg)
print_loss = 0.0
epoch_loss[0, epoch_idx] = print_epoch_loss / 15249
print_epoch_loss = 0.0
np_p1[0, epoch_idx] = p1_tmp
np_p5[0, epoch_idx] = p5_tmp
return epoch_loss, np_p1, np_p5
def signal_handler(signum, frame):
logger.info("Caught signal " + str(signum))
global requested_to_quit
requested_to_quit = True
def click_nowait(self):
x, y = self.all()[0].center()
logger.info("click %d, %d", x, y)
self._d.click(x, y)
def add_table_column2redis(pro_num,
*args,
column=None,
url_fun=None,
process_num=10,
fn="",
spider="",
module_name="",
class_name=None,
r_cfg=None,
m_cfg=None,
service=True,
from_id=None,
limit=3000,
**kwargs):
session = sa.SqlAlchemyHelper.get_session_by_cfg(m_cfg)
conn_redis = ur.RedisHelper.get_redis_connect_by_cfg(r_cfg)
if kwargs.get("del_q"):
conn_redis.delete(spider + kwargs.get("suffix", ":start_urls"))
if from_id is None:
from_id = conn_redis.get("kid_" + str(fn) + "_" + class_name +
"_" + str(pro_num) + "_from_id")
from_id = from_id if from_id else 0
logger.info("From id is \t" + str(from_id))
while 1:
if conn_redis.llen(spider +
kwargs.get("suffix", ":start_urls")) > limit:
BF.print_from_head("===Too much\t" + class_name + "\t")
time.sleep(2 * (pro_num + 1))
continue
model_class = getattr(importlib.import_module(module_name),
class_name)
infos = model_class.getByFromIdAndMod(from_id,
process_num,
pro_num,
session,
limit=10)
if infos:
for info in infos:
if url_fun:
url = url_fun(info.__dict__.get(column))
else:
url = info.__dict__.get(column)
if url:
url = url.strip()
if kwargs.get("bf"):
bf = BloomFilter.BloomFilter(conn_redis,
key=spider)
if bf.is_exists(url):
BF.print_no_end("-")
else:
res = conn_redis.lpush(
spider +
kwargs.get("suffix", ":start_urls"), url)
logger.info(str((spider, res, info.id, url)))
bf.add(url)
else:
res = conn_redis.lpush(
spider + kwargs.get("suffix", ":start_urls"),
url)
logger.info(str((spider, res, info.id, url)))
from_id = info.id
conn_redis.set(
"kid_" + str(fn) + "_" + class_name + "_" +
str(pro_num) + "_from_id", from_id)
else:
if service:
BF.print_from_head("No More\t" + class_name + "\t")
time.sleep(2 * (pro_num + 1))
session.commit()
else:
return
def _best_match(self, videos):
if not videos:
log.error("No videos found on YouTube for a given search")
return None
""" Select the best matching video from a list of videos. """
if const.args.manual:
log.info(self.raw_song)
log.info("0. Skip downloading this song.\n")
# fetch all video links on first page on YouTube
for i, v in enumerate(videos):
log.info(
u"{0}. {1} {2} {3}".format(
i + 1,
v["title"],
v["videotime"],
"http://youtube.com/watch?v=" + v["link"],
)
)
# let user select the song to download
result = internals.input_link(videos)
if result is None:
return None
else:
if not self.meta_tags:
# if the metadata could not be acquired, take the first result
# from Youtube because the proper song length is unknown
result = videos[0]
log.debug(
"Since no metadata found on Spotify, going with the first result"
)
else:
# filter out videos that do not have a similar length to the Spotify song
duration_tolerance = 10
max_duration_tolerance = 20
possible_videos_by_duration = []
# start with a reasonable duration_tolerance, and increment duration_tolerance
# until one of the Youtube results falls within the correct duration or
# the duration_tolerance has reached the max_duration_tolerance
while len(possible_videos_by_duration) == 0:
possible_videos_by_duration = list(
filter(
lambda x: abs(x["seconds"] - self.meta_tags["duration"])
<= duration_tolerance,
videos,
)
)
duration_tolerance += 1
if duration_tolerance > max_duration_tolerance:
log.error(
"{0} by {1} was not found.".format(
self.meta_tags["name"],
self.meta_tags["artists"][0]["name"],
)
)
return None
result = possible_videos_by_duration[0]
if result:
url = "http://youtube.com/watch?v={0}".format(result["link"])
else:
url = None
return url
def main():
parser = create_parser()
args = parser.parse_args()
with open(args.map) as fi:
mapping_pseudo_train = json.load(fi)
logger.info(args)
train_matrix_name = extract_matrix_name(args.embed_dir, args.train)
logger.info(train_matrix_name)
pseudo_matrix_name = extract_matrix_name(args.embed_dir, args.pseudo)
logger.info(pseudo_matrix_name)
train_matrix = h5py.File(train_matrix_name, "r")
pseudo_matrix = h5py.File(pseudo_matrix_name, "r")
original = 0
mask = 0
unmask = 0
mask_original = 0
unmask_original = 0
n_mask = 0
n_unmask = 0
len_file = length_fileobj(args.pseudo)
for idx, instance in tqdm(enumerate(read_instance(args.pseudo)),
total=len_file):
unique_id = str(instance["unique_id"])
# extract vector
train_vec = torch.Tensor(
train_matrix.get(mapping_pseudo_train[unique_id])[()])
pseudo_vec = torch.Tensor(pseudo_matrix.get(unique_id)[()])
mask_ids = [
idx for idx, token in enumerate(instance["surfaces"])
if token == MASK
]
unmask_ids = [
idx for idx, token in enumerate(instance["surfaces"])
if token != MASK
]
assert len(instance["surfaces"]) == len(mask_ids) + len(unmask_ids)
# sum
original += torch.sum(train_vec.norm(dim=1))
mask += torch.sum(pseudo_vec[mask_ids].norm(dim=1))
unmask += torch.sum(pseudo_vec[unmask_ids].norm(dim=1))
mask_original += torch.sum(
(train_vec[mask_ids] - pseudo_vec[mask_ids]).norm(dim=1))
unmask_original += torch.sum(
(train_vec[unmask_ids] - pseudo_vec[unmask_ids]).norm(dim=1))
n_mask += len(mask_ids)
n_unmask += len(unmask_ids)
if idx < 4:
logger.info("Iteration: {}".format(idx))
logger.debug("Original: {}".format(original / (n_mask + n_unmask)))
logger.debug("Mask: {}".format(mask / n_mask))
logger.debug("Unmask: {}".format(unmask / n_unmask))
logger.debug("Mask - Original: {}".format(mask_original / n_mask))
logger.debug("Unmask - Original: {}".format(unmask_original /
n_unmask))
logger.info(args.pseudo)
print("'Original': {},".format(original / (n_mask + n_unmask)))
print("'Mask': {},".format(mask / n_mask))
print("'Unmask': {},".format(unmask / n_unmask))
print("'Mask - Original': {},".format(mask_original / n_mask))
print("'Unmask - Original': {}".format(unmask_original / n_unmask))
# close
train_matrix.close()
pseudo_matrix.close()
def main():
logger.info("Reading data")
## read data
data = pd.read_csv('../inputs/bureau.csv')
### additional manual features
data['AMT_CREDIT_SUM_DEBT'] = data['AMT_CREDIT_SUM_DEBT'].fillna(0)
data['AMT_CREDIT_SUM_OVERDUE'] = data['AMT_CREDIT_SUM_OVERDUE'].fillna(0)
data['AMT_CREDIT_SUM_DEBT'] = data['AMT_CREDIT_SUM_DEBT'].fillna(0)
data['AMT_CREDIT_SUM'] = data['AMT_CREDIT_SUM'].fillna(0)
grp1 = data[['SK_ID_CURR', 'AMT_CREDIT_SUM_DEBT']].groupby(by = ['SK_ID_CURR'])['AMT_CREDIT_SUM_DEBT'].sum().reset_index().rename( index = str, columns = { 'AMT_CREDIT_SUM_DEBT': 'TOTAL_CUSTOMER_DEBT'})
grp2 = data[['SK_ID_CURR', 'AMT_CREDIT_SUM']].groupby(by = ['SK_ID_CURR'])['AMT_CREDIT_SUM'].sum().reset_index().rename( index = str, columns = { 'AMT_CREDIT_SUM': 'TOTAL_CUSTOMER_CREDIT'})
data = data.merge(grp1, on = ['SK_ID_CURR'], how = 'left')
data = data.merge(grp2, on = ['SK_ID_CURR'], how = 'left')
del grp1, grp2
data['DEBT_CREDIT_RATIO'] = data['TOTAL_CUSTOMER_DEBT']/data['TOTAL_CUSTOMER_CREDIT']
## amener au niveau client
debt_credit_ratio = data.groupby('SK_ID_CURR')['DEBT_CREDIT_RATIO'].mean().reset_index()
debt_credit_ratio.set_index('SK_ID_CURR', inplace=True, drop=False)
# take a sample
##data = data.sample(10000)
logger.info("calculating features")
## calcul features
if enable_time_features_expansion:
expand_time_features(data)
aggregation_groups = create_aggregation_groups()
vector_keys_mapping = data[vector_keys]
vector_keys_mapping = vector_keys_mapping.drop_duplicates()
compute_vector_key_features(data, aggregation_groups)
if enable_subtraction_features:
add_subtraction_features(vector_keys_mapping)
output_vector = vector_keys_mapping
for key, vector in output.items():
# Fix column names before final merge
key_prefix = 'key={0}|'.format(key)
vector = vector.add_prefix(key_prefix)
vector.reset_index(inplace=True)
output[key] = vector
output_vector = output_vector.merge(vector, how='left')
### final_merge
output_vector = output_vector.merge(debt_credit_ratio, how='left', on='SK_ID_CURR')
### fill na
output_vector.fillna(0)
logger.info("Finished...writing result vector to CSV")
### write it as csv
output_vector.to_csv('../outputs/azure_bureau_features.csv')
def Persist(self, block):
self._persisting_block = block
sn = self._db.snapshot()
accounts = DBCollection(self._db, sn, DBPrefix.ST_Account,
AccountState)
unspentcoins = DBCollection(self._db, sn, DBPrefix.ST_Coin,
UnspentCoinState)
spentcoins = DBCollection(self._db, sn, DBPrefix.ST_SpentCoin,
SpentCoinState)
assets = DBCollection(self._db, sn, DBPrefix.ST_Asset, AssetState)
validators = DBCollection(self._db, sn, DBPrefix.ST_Validator,
ValidatorState)
contracts = DBCollection(self._db, sn, DBPrefix.ST_Contract,
ContractState)
storages = DBCollection(self._db, sn, DBPrefix.ST_Storage, StorageItem)
amount_sysfee = self.GetSysFeeAmount(
block.PrevHash) + block.TotalFees().value
amount_sysfee_bytes = amount_sysfee.to_bytes(8, 'little')
to_dispatch = []
with self._db.write_batch() as wb:
wb.put(DBPrefix.DATA_Block + block.Hash.ToBytes(),
amount_sysfee_bytes + block.Trim())
for tx in block.Transactions:
wb.put(DBPrefix.DATA_Transaction + tx.Hash.ToBytes(),
block.IndexBytes() + tx.ToArray())
# go through all outputs and add unspent coins to them
unspentcoinstate = UnspentCoinState.FromTXOutputsConfirmed(
tx.outputs)
unspentcoins.Add(tx.Hash.ToBytes(), unspentcoinstate)
# go through all the accounts in the tx outputs
for output in tx.outputs:
account = accounts.GetAndChange(
output.AddressBytes, AccountState(output.ScriptHash))
if account.HasBalance(output.AssetId):
account.AddToBalance(output.AssetId, output.Value)
else:
account.SetBalanceFor(output.AssetId, output.Value)
# go through all tx inputs
unique_tx_input_hashes = []
for input in tx.inputs:
if input.PrevHash not in unique_tx_input_hashes:
unique_tx_input_hashes.append(input.PrevHash)
for txhash in unique_tx_input_hashes:
prevTx, height = self.GetTransaction(txhash.ToBytes())
coin_refs_by_hash = [
coinref for coinref in tx.inputs
if coinref.PrevHash.ToBytes() == txhash.ToBytes()
]
for input in coin_refs_by_hash:
uns = unspentcoins.GetAndChange(
input.PrevHash.ToBytes())
uns.OrEqValueForItemAt(input.PrevIndex,
CoinState.Spent)
if prevTx.outputs[input.PrevIndex].AssetId.ToBytes(
) == Blockchain.SystemShare().Hash.ToBytes():
sc = spentcoins.GetAndChange(
input.PrevHash.ToBytes(),
SpentCoinState(input.PrevHash, height, []))
sc.Items.append(
SpentCoinItem(input.PrevIndex, block.Index))
output = prevTx.outputs[input.PrevIndex]
acct = accounts.GetAndChange(
prevTx.outputs[input.PrevIndex].AddressBytes,
AccountState(output.ScriptHash))
assetid = prevTx.outputs[input.PrevIndex].AssetId
acct.SubtractFromBalance(
assetid, prevTx.outputs[input.PrevIndex].Value)
# do a whole lotta stuff with tx here...
if tx.Type == TransactionType.RegisterTransaction:
asset = AssetState(tx.Hash, tx.AssetType, tx.Name,
tx.Amount, Fixed8(0), tx.Precision,
Fixed8(0), Fixed8(0),
UInt160(data=bytearray(20)), tx.Owner,
tx.Admin, tx.Admin,
block.Index + 2 * 2000000, False)
assets.Add(tx.Hash.ToBytes(), asset)
elif tx.Type == TransactionType.IssueTransaction:
txresults = [
result for result in tx.GetTransactionResults()
if result.Amount.value < 0
]
for result in txresults:
asset = assets.GetAndChange(result.AssetId.ToBytes())
asset.Available = asset.Available - result.Amount
elif tx.Type == TransactionType.ClaimTransaction:
for input in tx.Claims:
sc = spentcoins.TryGet(input.PrevHash.ToBytes())
if sc and sc.HasIndex(input.PrevIndex):
sc.DeleteIndex(input.PrevIndex)
spentcoins.GetAndChange(input.PrevHash.ToBytes())
elif tx.Type == TransactionType.EnrollmentTransaction:
newvalidator = ValidatorState(pub_key=tx.PublicKey)
validators.GetAndChange(tx.PublicKey.ToBytes(),
newvalidator)
elif tx.Type == TransactionType.StateTransaction:
# @TODO Implement persistence for State Descriptors
pass
elif tx.Type == TransactionType.PublishTransaction:
contract = ContractState(tx.Code, tx.NeedStorage, tx.Name,
tx.CodeVersion, tx.Author,
tx.Email, tx.Description)
contracts.GetAndChange(tx.Code.ScriptHash().ToBytes(),
contract)
elif tx.Type == TransactionType.InvocationTransaction:
script_table = CachedScriptTable(contracts)
service = StateMachine(accounts, validators, assets,
contracts, storages, wb)
engine = ApplicationEngine(
trigger_type=TriggerType.Application,
container=tx,
table=script_table,
service=service,
gas=tx.Gas,
testMode=False)
engine.LoadScript(tx.Script, False)
try:
success = engine.Execute()
service.ExecutionCompleted(engine, success)
except Exception as e:
service.ExecutionCompleted(engine, False, e)
to_dispatch = to_dispatch + service.events_to_dispatch
else:
if tx.Type != b'\x00' and tx.Type != 128:
logger.info("TX Not Found %s " % tx.Type)
# do save all the accounts, unspent, coins, validators, assets, etc
# now sawe the current sys block
# filter out accounts to delete then commit
for key, account in accounts.Current.items():
if not account.IsFrozen and len(
account.Votes) == 0 and account.AllBalancesZeroOrLess(
):
accounts.Remove(key)
accounts.Commit(wb)
# filte out unspent coins to delete then commit
for key, unspent in unspentcoins.Current.items():
if unspent.IsAllSpent:
unspentcoins.Remove(key)
unspentcoins.Commit(wb)
# filter out spent coins to delete then commit to db
for key, spent in spentcoins.Current.items():
if len(spent.Items) == 0:
spentcoins.Remove(key)
spentcoins.Commit(wb)
# commit validators
validators.Commit(wb)
# commit assets
assets.Commit(wb)
# commit contracts
contracts.Commit(wb)
sn.close()
wb.put(DBPrefix.SYS_CurrentBlock,
block.Hash.ToBytes() + block.IndexBytes())
self._current_block_height = block.Index
self._persisting_block = None
for event in to_dispatch:
events.emit(event.event_type, event)
def __init__(self, path):
super(LevelDBBlockchain, self).__init__()
self._path = path
self._header_index = []
self._header_index.append(
Blockchain.GenesisBlock().Header.Hash.ToBytes())
try:
self._db = plyvel.DB(self._path, create_if_missing=True)
# self._db = plyvel.DB(self._path, create_if_missing=True, bloom_filter_bits=16, compression=None)
logger.info("Created Blockchain DB at %s " % self._path)
except Exception as e:
logger.info(
"leveldb unavailable, you may already be running this process: %s "
% e)
raise Exception('Leveldb Unavailable')
version = self._db.get(DBPrefix.SYS_Version)
if version == self._sysversion: # or in the future, if version doesn't equal the current version...
ba = bytearray(self._db.get(DBPrefix.SYS_CurrentBlock, 0))
self._current_block_height = int.from_bytes(ba[-4:], 'little')
ba = bytearray(self._db.get(DBPrefix.SYS_CurrentHeader, 0))
current_header_height = int.from_bytes(ba[-4:], 'little')
current_header_hash = bytes(ba[:64].decode('utf-8'),
encoding='utf-8')
# logger.info("current header hash!! %s " % current_header_hash)
# logger.info("current header height, hashes %s %s %s" %(self._current_block_height, self._header_index, current_header_height) )
hashes = []
try:
for key, value in self._db.iterator(
prefix=DBPrefix.IX_HeaderHashList):
ms = StreamManager.GetStream(value)
reader = BinaryReader(ms)
hlist = reader.Read2000256List()
key = int.from_bytes(key[-4:], 'little')
hashes.append({'k': key, 'v': hlist})
StreamManager.ReleaseStream(ms)
# hashes.append({'index':int.from_bytes(key, 'little'), 'hash':value})
except Exception as e:
logger.info("Could not get stored header hash list: %s " % e)
if len(hashes):
hashes.sort(key=lambda x: x['k'])
genstr = Blockchain.GenesisBlock().Hash.ToBytes()
for hlist in hashes:
for hash in hlist['v']:
if hash != genstr:
self._header_index.append(hash)
self._stored_header_count += 1
if self._stored_header_count == 0:
headers = []
for key, value in self._db.iterator(
prefix=DBPrefix.DATA_Block):
dbhash = bytearray(value)[8:]
headers.append(
Header.FromTrimmedData(binascii.unhexlify(dbhash), 0))
headers.sort(key=lambda h: h.Index)
for h in headers:
if h.Index > 0:
self._header_index.append(h.Hash.ToBytes())
elif current_header_height > self._stored_header_count:
try:
hash = current_header_hash
targethash = self._header_index[-1]
newhashes = []
while hash != targethash:
header = self.GetHeader(hash)
newhashes.insert(0, header)
hash = header.PrevHash.ToBytes()
self.AddHeaders(newhashes)
except Exception as e:
pass
else:
with self._db.write_batch() as wb:
for key, value in self._db.iterator():
wb.delete(key)
self.Persist(Blockchain.GenesisBlock())
self._db.put(DBPrefix.SYS_Version, self._sysversion)
def main(data_cnf, model_cnf, mode):
model_name = os.path.split(model_cnf)[1].split(".")[0]
yaml = YAML(typ='safe')
data_cnf, model_cnf = yaml.load(Path(data_cnf)), yaml.load(Path(model_cnf))
# 設定log檔案位置
logfile("./logs/logfile_{0}_cornet_{1}_cornet_dim_{2}.log".format(
model_name, model_cnf['model']['n_cornet_blocks'],
model_cnf['model']['cornet_dim']))
model, model_name, data_name = None, model_cnf['name'], data_cnf['name']
model_path = os.path.join(
model_cnf['path'],
F'{model_name}-{data_name}-{model_cnf["model"]["n_cornet_blocks"]}-{model_cnf["model"]["cornet_dim"]}'
)
emb_init = get_word_emb(data_cnf['embedding']['emb_init'])
logger.info(F'Model Name: {model_name}')
# summary(model_dict[model_name])
if mode is None or mode == 'train':
logger.info('Loading Training and Validation Set')
train_x, train_labels = get_data(data_cnf['train']['texts'],
data_cnf['train']['labels'])
if 'size' in data_cnf['valid']:
random_state = data_cnf['valid'].get('random_state', 1240)
train_x, valid_x, train_labels, valid_labels = train_test_split(
train_x,
train_labels,
test_size=data_cnf['valid']['size'],
random_state=random_state)
else:
valid_x, valid_labels = get_data(data_cnf['valid']['texts'],
data_cnf['valid']['labels'])
mlb = get_mlb(data_cnf['labels_binarizer'],
np.hstack((train_labels, valid_labels)))
train_y, valid_y = mlb.transform(train_labels), mlb.transform(
valid_labels)
labels_num = len(mlb.classes_)
logger.info(F'Number of Labels: {labels_num}')
logger.info(F'Size of Training Set: {len(train_x)}')
logger.info(F'Size of Validation Set: {len(valid_x)}')
logger.info('Training')
train_loader = DataLoader(MultiLabelDataset(train_x, train_y),
model_cnf['train']['batch_size'],
shuffle=True,
num_workers=4)
valid_loader = DataLoader(MultiLabelDataset(valid_x,
valid_y,
training=True),
model_cnf['valid']['batch_size'],
num_workers=4)
if 'gpipe' not in model_cnf:
model = Model(network=model_dict[model_name],
labels_num=labels_num,
model_path=model_path,
emb_init=emb_init,
**data_cnf['model'],
**model_cnf['model'])
else:
model = GPipeModel(model_name,
labels_num=labels_num,
model_path=model_path,
emb_init=emb_init,
**data_cnf['model'],
**model_cnf['model'])
loss, p1, p5 = model.train(train_loader, valid_loader,
**model_cnf['train'])
np.save(
model_cnf['np_loss'] + "{0}_cornet_{1}_cornet_dim_{2}.npy".format(
model_name, model_cnf['model']['n_cornet_blocks'],
model_cnf['model']['cornet_dim']), loss)
np.save(
model_cnf['np_p1'] + "{0}_cornet_{1}_cornet_dim_{2}.npy".format(
model_name, model_cnf['model']['n_cornet_blocks'],
model_cnf['model']['cornet_dim']), p1)
np.save(
model_cnf['np_p5'] + "{0}_cornet_{1}_cornet_dim_{2}.npy".format(
model_name, model_cnf['model']['n_cornet_blocks'],
model_cnf['model']['cornet_dim']), p5)
logger.info('Finish Training')
if mode is None or mode == 'eval':
logger.info('Loading Test Set')
logger.info('model path: ', model_path)
mlb = get_mlb(data_cnf['labels_binarizer'])
labels_num = len(mlb.classes_)
test_x, _ = get_data(data_cnf['test']['texts'], None)
logger.info(F'Size of Test Set: {len(test_x)}')
logger.info('Predicting')
test_loader = DataLoader(MultiLabelDataset(test_x),
model_cnf['predict']['batch_size'],
num_workers=4)
if 'gpipe' not in model_cnf:
if model is None:
model = Model(network=model_dict[model_name],
labels_num=labels_num,
model_path=model_path,
emb_init=emb_init,
**data_cnf['model'],
**model_cnf['model'])
else:
if model is None:
model = GPipeModel(model_name,
labels_num=labels_num,
model_path=model_path,
emb_init=emb_init,
**data_cnf['model'],
**model_cnf['model'])
scores, labels = model.predict(test_loader,
k=model_cnf['predict'].get('k', 3801))
logger.info('Finish Predicting')
labels = mlb.classes_[labels]
output_res(data_cnf['output']['res'], F'{model_name}-{data_name}',
scores, labels)
def main(data_cnf, model_cnf, mode):
model_name = os.path.split(model_cnf)[1].split(".")[0]
# 設定log檔案位置
logfile("./logs/logfile_" + model_name + ".log")
yaml = YAML(typ='safe')
data_cnf, model_cnf = yaml.load(Path(data_cnf)), yaml.load(Path(model_cnf))
model, model_name, data_name = None, model_cnf['name'], data_cnf['name']
# model_path = model_cnf['path'] + "/" + model_cnf['name'] + '.h'
model_path = r'E:\\PycharmProject\\CorNet\\' + model_name + '.h5'
emb_init = get_word_emb(data_cnf['embedding']['emb_init'])
logger.info(F'Model Name: {model_name}')
# keras log file
csv_logger = CSVLogger('./logs/' + model_name + '_log.csv', append=True, separator=',')
if mode is None or mode == 'train':
logger.info('Loading Training and Validation Set')
train_x, train_labels = get_data(data_cnf['train']['texts'], data_cnf['train']['labels'])
if 'size' in data_cnf['valid']:
random_state = data_cnf['valid'].get('random_state', 1240)
train_x, valid_x, train_labels, valid_labels = train_test_split(train_x, train_labels,
test_size=data_cnf['valid']['size'],
random_state=random_state)
else:
valid_x, valid_labels = get_data(data_cnf['valid']['texts'], data_cnf['valid']['labels'])
mlb = get_mlb(data_cnf['labels_binarizer'], np.hstack((train_labels, valid_labels)))
train_y, valid_y = mlb.transform(train_labels), mlb.transform(valid_labels)
labels_num = len(mlb.classes_)
logger.info(F'Number of Labels: {labels_num}')
logger.info(F'Size of Training Set: {len(train_x)}')
logger.info(F'Size of Validation Set: {len(valid_x)}')
vocab_size = emb_init.shape[0]
emb_size = emb_init.shape[1]
# 可調參數
data_num = len(train_x)
ks = 3
output_channel = model_cnf['model']['num_filters']
dynamic_pool_length = model_cnf['model']['dynamic_pool_length']
num_bottleneck_hidden = model_cnf['model']['bottleneck_dim']
drop_out = model_cnf['model']['dropout']
cornet_dim = model_cnf['model']['cornet_dim']
nb_cornet_block = model_cnf['model'].get('nb_cornet_block', 0)
nb_epochs = model_cnf['train']['nb_epoch']
batch_size = model_cnf['train']['batch_size']
max_length = 500
input_tensor = Input(batch_shape=(batch_size, max_length), name='input')
emb_data = Embedding(input_dim=vocab_size,
output_dim=emb_size,
input_length=max_length,
weights=[emb_init],
trainable=False,
name='embedding1')(input_tensor)
emb_data.trainable = False
# emd_out_4d = keras.layers.core.RepeatVector(1)(emb_data)
# unsqueeze_emb_data = tf.keras.layers.Reshape((1, 500, 300), input_shape=(500, 300))(emb_data)
# emb_data = tf.expand_dims(emb_data, axis=1)
# emb_data = Lambda(reshape_tensor, arguments={'shape': (1, max_length, 300)}, name='lambda1')(
# emb_data)
conv1_output = Convolution1D(output_channel, 2, padding='same',
kernel_initializer=keras.initializers.glorot_uniform(seed=None),
activation='relu', name='conv1')(emb_data)
# conv1_output = Lambda(reshape_tensor, arguments={'shape': (batch_size, max_length, output_channel)},
# name='conv1_lambda')(
# conv1_output)
conv2_output = Convolution1D(output_channel, 4, padding='same',
kernel_initializer=keras.initializers.glorot_uniform(seed=None),
activation='relu', name='conv2')(emb_data)
# conv2_output = Lambda(reshape_tensor, arguments={'shape': (batch_size, max_length, output_channel)},
# name='conv2_lambda')(
# conv2_output)
conv3_output = Convolution1D(output_channel, 8, padding='same',
kernel_initializer=keras.initializers.glorot_uniform(seed=None),
activation='relu', name='conv3')(emb_data)
# conv3_output = Lambda(reshape_tensor, arguments={'shape': (batch_size, max_length, output_channel)},
# name='conv3_lambda')(
# conv3_output)
# pool1 = adapmaxpooling(conv1_output, dynamic_pool_length)
pool1 = GlobalMaxPooling1D(name='globalmaxpooling1')(conv1_output)
pool2 = GlobalMaxPooling1D(name='globalmaxpooling2')(conv2_output)
pool3 = GlobalMaxPooling1D(name='globalmaxpooling3')(conv3_output)
output = concatenate([pool1, pool2, pool3], axis=-1)
# output = Dense(num_bottleneck_hidden, activation='relu',name='bottleneck')(output)
output = Dropout(drop_out, name='dropout1')(output)
output = Dense(labels_num, activation='softmax', name='dense_final',
kernel_initializer=keras.initializers.glorot_uniform(seed=None))(output)
if nb_cornet_block > 0:
for i in range(nb_cornet_block):
x_shortcut = output
x = keras.layers.Activation('sigmoid', name='cornet_sigmoid_{0}'.format(i + 1))(output)
x = Dense(cornet_dim, kernel_initializer='glorot_uniform', name='cornet_1st_dense_{0}'.format(i + 1))(x)
# x = Dense(cornet_dim, kernel_initializer=keras.initializers.glorot_uniform(seed=None),
# activation='sigmoid', name='cornet_1st_dense_{0}'.format(i + 1))(output)
x = keras.layers.Activation('elu', name='cornet_elu_{0}'.format(i + 1))(x)
x = Dense(labels_num, kernel_initializer='glorot_uniform', name='cornet_2nd_dense_{0}'.format(i + 1))(x)
# x = Dense(labels_num, kernel_initializer=keras.initializers.glorot_uniform(seed=None), activation='elu',
# name='cornet_2nd_dense_{0}'.format(i + 1))(x)
output = Add()([x, x_shortcut])
model = Model(input_tensor, output)
model.summary()
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=[tf.keras.metrics.Precision(top_k=5)])
# model.compile(optimizer='adam', loss='binary_crossentropy', metrics=[tf.keras.metrics.top_k_categorical_accuracy(k=5)])
model.fit_generator(steps_per_epoch=data_num / batch_size,
generator=batch_generator(train_x, train_y, batch_size),
validation_data=batch_generator(valid_x, valid_y, batch_size),
validation_steps=valid_x.shape[0] / batch_size,
nb_epoch=nb_epochs, callbacks=[csv_logger])
model.save(model_path)
elif mode is None or mode == 'eval':
logger.info('Loading Training and Validation Set')
train_x, train_labels = get_data(data_cnf['train']['texts'], data_cnf['train']['labels'])
if 'size' in data_cnf['valid']: # 如果有設定valid的size 則直接使用train的一部分作為valid
random_state = data_cnf['valid'].get('random_state', 1240)
train_x, valid_x, train_labels, valid_labels = train_test_split(train_x, train_labels,
test_size=data_cnf['valid']['size'],
random_state=random_state)
else:
valid_x, valid_labels = get_data(data_cnf['valid']['texts'], data_cnf['valid']['labels'])
mlb = get_mlb(data_cnf['labels_binarizer'], np.hstack((train_labels, valid_labels)))
train_y, valid_y = mlb.transform(train_labels), mlb.transform(valid_labels)
labels_num = len(mlb.classes_)
##################################################################################################
logger.info('Loading Test Set')
logger.info('model path: ', model_path)
mlb = get_mlb(data_cnf['labels_binarizer'])
labels_num = len(mlb.classes_)
test_x, test_label = get_data(data_cnf['test']['texts'], data_cnf['test']['labels'])
logger.info(F'Size of Test Set: {len(test_x)}')
test_y = mlb.transform(test_label).toarray()
model = tf.keras.models.load_model(model_path)
score = model.predict(test_x)
print("p5: ", p5(test_y, score))
def wdsync_folder(mytable=None, checkonly=False):
logger.info(f"test SyncMySQL().new_folder_in_243_not_241()...")
status, dirlist = SyncMySQL().new_folder_in_243_not_241(roottable=mytable)
if status:
dirstr = '\n'.join(sorted(dirlist))
logger.info(f"New folder {len(dirlist)} includes:\n{dirstr}")
logger.info(len(dirlist))
if checkonly:
return True
for elm in sorted(dirlist):
mydir = elm.replace('/mnt', '')
sqmydir = mydir.replace("'", "''")
logger.info(f"test sync {mydir}...")
if not dir_sync(mydir=sqmydir):
logger.error(f"Failed to sync {mydir}!!!")
return False
logger.info(f"test SyncMySQL().same_folder_in_243_and_241()...")
status, dirlist = SyncMySQL().same_folder_in_243_and_241(roottable=mytable)
if status:
dirstr = '\n'.join(sorted(dirlist))
logger.info(f"Same folder {len(dirlist)} includes:\n{dirstr}")
for elm in sorted(dirlist):
mydir = elm.replace('/mnt', '')
sqmydir = mydir.replace("'", "''")
logger.info(f"test sync {mydir}...")
if not dir_sync(mydir=sqmydir):
logger.error(f"Failed to sync {mydir}!!!")
return False
return True
wdsync_folder(mytable='data', checkonly=False)
wdsync_folder(mytable='photos', checkonly=False)
wdsync_folder(mytable='music', checkonly=False)
'''
mytbl = 'music' # 'data' # 'photos' #
logger.info(f"Get filelist from {mytbl}...")
# status, rst=SyncMySQL().compare_new_file_in_243_not_241(mytable=mytbl)
# if not status:
# return False
# logger.info(f"There are {len(rst)} new files in {mytbl}.")
status, rst = SyncMySQL().compare_diff_file_in_243_not_241(mytable=mytbl)
if not status:
return False
logger.info(f"There are {len(rst)} differentfiles in {mytbl}.")
if len(rst) == 0:
return True
status = filelist_sync(filelist=rst)
if not status:
logger.error(f"Failed to sync {mytbl} using filelist!!!s")
return True
if __name__ == '__main__':
logger.info(f"Start python code file {__file__}.")
if len(sys.argv) > 1 and 'nodebug' in sys.argv:
logzero.loglevel(logging.INFO)
if not main():
logger.error(f"Failed to call main")
exit(1)
logger.info(f"Success python code file {__file__}.")
def setup_signal_handling():
logger.info("setting up signal handling")
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)
def ensure_experiment_is_valid(experiment: Experiment):
"""
A chaos experiment consists of a method made of activities to carry
sequentially.
There are two kinds of activities:
* probe: detecting the state of a resource in your system or external to it
There are two kinds of probes: `steady` and `close`
* action: an operation to apply against your system
Usually, an experiment is made of a set of `steady` probes that ensure the
system is sound to carry further the experiment. Then, an action before
another set of of ̀close` probes to sense the state of the system
post-action.
This function raises :exc:`InvalidExperiment`, :exc:`InvalidProbe` or
:exc:`InvalidAction` depending on where it fails.
"""
logger.info("Validating the experiment's syntax")
if not experiment:
raise InvalidExperiment("an empty experiment is not an experiment")
if not experiment.get("title"):
raise InvalidExperiment("experiment requires a title")
if not experiment.get("description"):
raise InvalidExperiment("experiment requires a description")
tags = experiment.get("tags")
if tags:
if list(filter(lambda t: t == '' or not isinstance(t, str), tags)):
raise InvalidExperiment(
"experiment tags must be a non-empty string")
validate_extensions(experiment)
config = load_configuration(experiment.get("configuration", {}))
secrets = load_secrets(experiment.get("secrets", {}), config)
ensure_hypothesis_is_valid(experiment)
method = experiment.get("method")
if not method:
raise InvalidExperiment("an experiment requires a method with "
"at least one activity")
for activity in method:
ensure_activity_is_valid(activity)
# let's see if a ref is indeed found in the experiment
ref = activity.get("ref")
if ref and not lookup_activity(ref):
raise InvalidActivity("referenced activity '{r}' could not be "
"found in the experiment".format(r=ref))
rollbacks = experiment.get("rollbacks", [])
for activity in rollbacks:
ensure_activity_is_valid(activity)
warn_about_deprecated_features(experiment)
logger.info("Experiment looks valid")
def main():
# Training settings
parser = argparse.ArgumentParser(description='A modified version of Pytorch MNIST example to classify kanji characters')
parser.add_argument('--batch-size', type=int, default=64, metavar='N',
help='input batch size for training (default: 64)')
parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N',
help='input batch size for testing (default: 1000)')
parser.add_argument('--epochs', type=int, default=20, metavar='N',
help='number of epochs to train (default: 10)')
parser.add_argument('--lr', type=float, default=0.01, metavar='LR',
help='learning rate (default: 0.01)')
parser.add_argument('--momentum', type=float, default=0.5, metavar='M',
help='SGD momentum (default: 0.5)')
parser.add_argument('--no-cuda', action='store_true', default=False,
help='disables CUDA training')
parser.add_argument('--seed', type=int, default=1, metavar='S',
help='random seed (default: 1)')
parser.add_argument('--log-interval', type=int, default=100, metavar='N',
help='how many batches to wait before logging training status')
parser.add_argument('--save-model-each-epoch', action='store_true', default=False,
help='Save the model each epoch')
parser.add_argument('--root-dir', type=str, default='.',
help='path for dataset where data is stored in structured manner')
args = parser.parse_args()
use_cuda = not args.no_cuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
device = torch.device('cuda' if use_cuda else 'cpu')
# kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
logger.info('loading dataset...')
# perhaps some normalization is necessary...
train_data, dev_data, idx2cls = read_dataset(args.root_dir, seed=args.seed, transform=transforms.ToTensor())
train_loader = torch.utils.data.DataLoader(train_data, batch_size=args.batch_size, shuffle=True)
dev_loader = torch.utils.data.DataLoader(dev_data, batch_size=args.test_batch_size, shuffle=False)
logger.info('dataset preparation completed')
params = args.__dict__
params['idx2cls'] = idx2cls
# save parameters
with open('./params.json', 'w') as f:
json.dump(params, f)
model = Net(len(idx2cls)).to(device)
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)
min_test_loss = sys.float_info.max
train_start_time = time.time()
for epoch in range(1, args.epochs + 1):
train(args, model, device, train_loader, optimizer, epoch)
dev_loss = test(args, model, device, dev_loader)
if args.save_model_each_epoch:
torch.save(model.state_dict(), 'kanji_recognizer_ep{}.pt'.format(epoch))
logger.info('saved checkpoint: ep{}'.format(epoch))
if dev_loss < min_test_loss:
torch.save(model.state_dict(), 'kanji_recognizer_best.pt')
logger.info('best model now updated')
min_test_loss = dev_loss
torch.save(model.state_dict(), 'kanji_recognizer_last.pt')
elapsed_time = time.time() - train_start_time
logger.info('training finished in {} seconds'.format(round(elapsed_time)))
def run_experiment(experiment: Experiment,
settings: Settings = None) -> Journal:
"""
Run the given `experiment` method step by step, in the following sequence:
steady probe, action, close probe.
Activities can be executed in background when they have the
`"background"` property set to `true`. In that case, the activity is run in
a thread. By the end of runs, those threads block until they are all
complete.
If the experiment has the `"dry"` property set to `False`, the experiment
runs without actually executing the activities.
NOTE: Tricky to make a decision whether we should rollback when exiting
abnormally (Ctrl-C, SIGTERM...). Afterall, there is a chance we actually
cannot afford to rollback properly. Better bailing to a conservative
approach. This means we swallow :exc:`KeyboardInterrupt` and
:exc:`SystemExit` and do not bubble it back up to the caller. We when were
interrupted, we set the `interrupted` flag of the result accordingly to
notify the caller this was indeed not terminated properly.
"""
logger.info("Running experiment: {t}".format(t=experiment["title"]))
dry = experiment.get("dry", False)
if dry:
logger.warning("Dry mode enabled")
started_at = time.time()
config = load_configuration(experiment.get("configuration", {}))
secrets = load_secrets(experiment.get("secrets", {}), config)
activity_pool, rollback_pool = get_background_pools(experiment)
journal = initialize_run_journal(experiment)
try:
# this may fail the entire experiment right there if any of the probes
# fail or fall out of their tolerance zone
try:
state = run_steady_state_hypothesis(experiment, config, secrets,
dry)
journal["steady_states"]["before"] = state
if state is not None and not state["steady_state_met"]:
p = state["probes"][-1]
raise ActivityFailed(
"Steady state probe '{p}' is not in the given tolerance "
"so failing this experiment".format(
p=p["activity"]["name"]))
except ActivityFailed as a:
journal["steady_states"]["before"] = state
journal["status"] = "failed"
logger.fatal(str(a))
else:
try:
journal["run"] = apply_activities(experiment, config, secrets,
activity_pool, dry)
except Exception as x:
journal["status"] = "aborted"
logger.fatal(
"Experiment ran into an un expected fatal error, "
"aborting now.",
exc_info=True)
else:
try:
state = run_steady_state_hypothesis(
experiment, config, secrets, dry)
journal["steady_states"]["after"] = state
if state is not None and not state["steady_state_met"]:
journal["deviated"] = True
p = state["probes"][-1]
raise ActivityFailed(
"Steady state probe '{p}' is not in the given "
"tolerance so failing this experiment".format(
p=p["activity"]["name"]))
except ActivityFailed as a:
journal["status"] = "failed"
logger.fatal(str(a))
except (KeyboardInterrupt, SystemExit):
journal["status"] = "interrupted"
logger.warn("Received an exit signal, "
"leaving without applying rollbacks.")
else:
journal["status"] = journal["status"] or "completed"
journal["rollbacks"] = apply_rollbacks(experiment, config, secrets,
rollback_pool, dry)
journal["end"] = datetime.utcnow().isoformat()
journal["duration"] = time.time() - started_at
has_deviated = journal["deviated"]
status = "deviated" if has_deviated else journal["status"]
logger.info("Experiment ended with status: {s}".format(s=status))
if has_deviated:
logger.info("The steady-state has deviated, a weakness may have been "
"discovered")
return journal
parser.add_argument('--print_banner', type=int, nargs='?',
default=1,
help='is print banner param. input int 1 or 0')
args = parser.parse_args()
if args.print_banner:
print_app_info()
broker_hosts = args.broker_hosts
topic_name = args.topic_name
group_id = args.group_id
g_resource.OUT_PATH = args.out_path
log.info('##### kafka config')
log.info('# broker_hosts : {0}'.format(', '.join(broker_hosts)))
log.info('# topic_name : {0}'.format(topic_name))
log.info('# group_id : {0}'.format(group_id))
log.info('# out_path : {0}'.format(g_resource.OUT_PATH))
sys.argv.clear()
sys.argv.append(zero_arg)
sys.argv = sys.argv + fire_argv
log.debug('fire sys.argv : {0}'.format(', '.join(sys.argv)))
file_pipe = None
tar_stream = None
consumer = None
actionFactory = ActionFactory()
import datetime
from functools import wraps
import logzero
from logzero import logger
# Setup rotating logfile with 3 rotations, each with a maximum filesize of 1MB:
logzero.logfile("logfile.log", maxBytes=2048, backupCount=3)
# Log messages
def timer(f):
@wraps(f)
def wrapped(*args, **kwargs):
start = datetime.datetime.now()
r = f(*args, **kwargs)
print((datetime.datetime.now() - start).microseconds)
return r
return wrapped
if __name__ == '__main__':
logger.info("This log message goes to the console and the logfile")
def GetArray(self):
logger.info("trying to get array:: %s " % self)
raise Exception('Not supported')
def swa_init(self):
if 'swa' not in self.state:
logger.info('SWA Initializing')
swa_state = self.state['swa'] = {'models_num': 1}
for n, p in self.model.named_parameters():
swa_state[n] = p.data.cpu().detach()
def _overlay_labels(first_stage_reg_dir: Path,
inverted_labeldir: Path,
out_dir_labels: Path,
mask: Path = None):
"""
Overlay the first registrated image (rigid) with the corresponding inverted labels
It depends on the registered volumes and inverted label maps being named identically
"""
if mask:
mask = sitk.GetArrayFromImage(sitk.ReadImage(str(mask)))
rp = regionprops(mask)
# Get the largest label. Likley only one from the mask
mask_props = list(reversed(sorted(rp, key=lambda x: x.area)))[0]
bbox = mask_props['bbox']
for vol_path in common.get_file_paths(
first_stage_reg_dir,
ignore_folders=[RESOLUTION_IMGS_DIR, IMG_PYRAMID_DIR]):
vol_reader = common.LoadImage(vol_path)
if not vol_reader:
logging.error(f'cannnot create qc image from {vol_path}')
return
label_path = inverted_labeldir / vol_path.stem / vol_path.name
if label_path.is_file():
label_reader = common.LoadImage(label_path)
if not label_reader:
logging.error(
f'cannot create qc image from label file {label_path}')
return
cast_img = sitk.Cast(sitk.RescaleIntensity(vol_reader.img),
sitk.sitkUInt8)
arr = sitk.GetArrayFromImage(cast_img)
base = splitext(basename(label_reader.img_path))[0]
l_arr = label_reader.array
def sag(idx_):
slice_sag = np.flipud(arr[:, :, idx_])
l_slice_sag = np.flipud(l_arr[:, :, idx_])
sag_dir = out_dir_labels / 'sagittal'
sag_dir.mkdir(exist_ok=True)
out_path_sag = sag_dir / f'{base}_{idx_}.png'
_blend_8bit(slice_sag, l_slice_sag, out_path_sag)
if mask is None: # get a few slices from middle
sag_indxs = np.linspace(0, arr.shape[2], 8, dtype=np.int)[2:-2]
else:
sag_start = bbox[2]
sag_end = bbox[5]
sag_indxs = np.linspace(
sag_start, sag_end, 6,
dtype=np.int)[1:-1] # Take the 4 inner slices
for idx in sag_indxs:
sag(idx)
def ax(idx_):
slice_ax = arr[idx_, :, :]
l_slice_ax = l_arr[idx_, :, :]
ax_dir = out_dir_labels / 'axial'
ax_dir.mkdir(exist_ok=True)
out_path_ax = ax_dir / f'{base}_{idx_}.png'
_blend_8bit(slice_ax, l_slice_ax, out_path_ax)
if mask is None: # get a few slices from middle
ax_indxs = np.linspace(0, arr.shape[0], 8, dtype=np.int)[2:-2]
else:
ax_start = bbox[0]
ax_end = bbox[3]
ax_indxs = np.linspace(ax_start, ax_end, 6, dtype=np.int)[1:-1]
for idx in ax_indxs:
ax(idx)
def cor(idx_):
slice_cor = np.flipud(arr[:, idx_, :])
l_slice_cor = np.flipud(l_arr[:, idx_, :])
cor_dir = out_dir_labels / 'coronal'
cor_dir.mkdir(exist_ok=True)
out_path_cor = cor_dir / f'{base}_{idx_}.png'
_blend_8bit(slice_cor, l_slice_cor, out_path_cor)
if mask is None: # get a few slices from middle
cor_indxs = np.linspace(0, arr.shape[1], 8, dtype=np.int)[2:-2]
else:
cor_start = bbox[1]
cor_end = bbox[4]
cor_indxs = np.linspace(cor_start, cor_end, 6,
dtype=np.int)[1:-1]
for idx in cor_indxs:
cor(idx)
else:
logging.info(
'No inverted label found. Skipping creation of inverted label-image overlay'
)
def main(args=None):
ores = read_ores_json()
minerals = [f.name for f in dataclasses.fields(Minerals)]
all_orders = []
ALL_ORDERS_JSON = 'all_orders.cache.json'
if os.path.exists(ALL_ORDERS_JSON) and time.time() - os.path.getmtime(
ALL_ORDERS_JSON) < 3600:
logger.info('using cached orders')
with open(ALL_ORDERS_JSON, 'r') as f:
all_orders = Order.schema().loads(f.read(), many=True)
else:
logger.info('downloading orders')
for oreitem in [o.item_id for o in ores.values()]:
all_orders.extend(get_orders(oreitem))
with open(ALL_ORDERS_JSON, 'w+') as f:
f.write(Order.schema().dumps(all_orders, many=True))
names = {o.item_id: n for (n, o) in ores.items()}
rounded_orders = {}
for o in all_orders:
r = o
r.price = round(r.price, 0)
k = f'{names[r.item_id]} at {math.ceil(r.price):,}'
if k in rounded_orders:
rounded_orders[k].volume_remain += r.volume_remain
else:
rounded_orders[k] = r
solver = pywraplp.Solver('minerals',
pywraplp.Solver.GLOP_LINEAR_PROGRAMMING)
order_vars = {
k: solver.NumVar(0, v.volume_remain, k)
for (k, v) in rounded_orders.items()
}
# Objective function: minimize cost of what we buy.
objective = solver.Objective()
for k, var in order_vars.items():
objective.SetCoefficient(var, rounded_orders[k].price)
objective.SetMinimization()
# \!/ Problem specification - you want to edit this part.
for_phoons = Minerals(2.5e6, 9.9e6, 1e6)
twenty_thoraxes = Minerals(10.3e6, 1.29e6, 0.35e6, 94.06e3, 26.73e3,
10.11e3, 2567)
twenty_moas = 2 * Minerals(5.45e6, 1.39e6, 0.3564e6, 88.12e3, 23.77e3,
11.28e3, 3458)
twenty_stabbers = 2 * Minerals(4.56e6, 1.683e6, 0.3564e6, 92.07e3,
24.752e3, 11.09e3, 2175)
ten_tornadoes = Minerals(39.06e3, 9.744e6, 2.354e6, .6972e6, 0, 84592,
32914)
ten_thrashers = Minerals(431163, 103347, 35793, 15813, 0, 1620, 234)
STASH = Minerals(isogen=13077093,
nocxium=2922802,
zydrine=607902,
mexallon=2070878,
megacyte=249423)
TARGET = 2 * Minerals(*dataclasses.astuple(
twenty_thoraxes + twenty_moas + twenty_stabbers)[0:3]) + Minerals(
pyerite=20e6, mexallon=2e6) + Minerals(tritanium=40e6)
TARGET = 2 * ten_tornadoes + 2 * ten_thrashers + twenty_stabbers + -1 * STASH
# \!/ Problem specification - you want to edit this part.
for mineral in minerals:
# The ores we buy need to refine into at least TARGET[mineral] many units.
ct = solver.Constraint(getattr(TARGET, mineral), solver.infinity())
for k, var in order_vars.items():
ct.SetCoefficient(
var,
getattr(ores[names[rounded_orders[k].item_id]].refines_to,
mineral))
status = solver.Solve()
print("Status: ", status, status == solver.OPTIMAL and "OPTIMAL" or "",
status == solver.FEASIBLE and "SUB-OPTIMAL BUT FEASIBLE" or "")
if status in [solver.OPTIMAL, solver.FEASIBLE]:
price = 0
volume = 0
to_buy = defaultdict(float)
for k, var in order_vars.items():
if var.solution_value() != 0:
price += rounded_orders[k].price * var.solution_value()
volume += ores[names[
rounded_orders[k].item_id]].volume * var.solution_value()
print(
f'{var.solution_value()}x {k} ({rounded_orders[k].volume_remain:,} available)'
)
to_buy[names[
rounded_orders[k].item_id]] += var.solution_value()
print("\n\n")
# now kronch that down from per-order into per-ore
to_buy = {k: math.ceil(v) for k, v in to_buy.items()}
refined_minerals = Minerals()
for k, v in to_buy.items():
refined_minerals += ores[k].refines_to * v
print("Yielding:")
for k, v in dataclasses.asdict(refined_minerals).items():
print(f'{v:,}x {k} (excess of {v - getattr(TARGET, k):,})')
print("\n\nINPUT TO MULTIBUY:\n")
for k, v in to_buy.items():
print(f'{v}x {k}')
print(f"\n\nTotal price: {math.ceil(price):,}")
print(f"Total volume: {volume:,} m3")
def train():
with open("config.toml") as f:
config = toml.load(f)
base_dir = config["data"]["base_dir"]
epochs = config["train"]["epochs"]
batch_size = config["train"]["batch_size"]
lr = config["train"]["lr"]
betas = config["train"]["betas"]
in_filters = config["model"]["in_filters"]
image_size = config["model"]["image_size"]
filters = config["model"]["filters"]
num_classes = config["model"]["num_classes"]
kernel_size = config["model"]["kernel_size"]
padding = config["model"]["padding"]
num_resblocks = config["model"]["num_resblocks"]
device = "cuda" if torch.cuda.is_available() else "cpu"
records = load_data(base_dir)
train_records, test_records = train_test_split(records, test_size=0.2)
train_transform = transforms.Compose([
transforms.Resize(image_size),
transforms.RandomAffine(10, translate=[0.1, 0.1], shear=0.1),
transforms.ColorJitter(brightness=0.7, contrast=0.7),
transforms.ToTensor(),
transforms.Normalize(0.5, 0.5)
])
test_transform = transforms.Compose([
transforms.Resize(image_size),
transforms.ToTensor(),
transforms.Normalize(0.5, 0.5)
])
trainset = CovidChestxrayDataset(train_records, base_dir, train_transform)
testset = CovidChestxrayDataset(test_records, base_dir, test_transform)
trainloader = DataLoader(trainset, batch_size=batch_size, shuffle=True)
testloader = DataLoader(testset, batch_size=1, shuffle=False)
net = Model(in_filters, image_size, filters, kernel_size, padding,
num_resblocks, num_classes)
net.to(device)
criterion = nn.NLLLoss()
optimizer = optim.AdamW(net.parameters(),
lr=lr,
betas=betas,
weight_decay=1e-2)
for epoch in range(epochs):
net.train()
train_loss = 0
train_targets = []
train_probs = []
train_preds = []
grad = 0
for batch in trainloader:
img, label = batch
train_targets += label.numpy().tolist()
img, label = img.to(device), label.to(device)
optimizer.zero_grad()
pred = net(img)
loss = criterion(pred, label)
loss.backward()
grad += check_grad(net.parameters())
torch.nn.utils.clip_grad_norm_(net.parameters(), 1)
optimizer.step()
train_loss += loss.item()
train_preds += pred.cpu().detach().numpy().argmax(axis=1).tolist()
train_probs += pred.cpu().detach().numpy()[:, 1].tolist()
acc = accuracy_score(train_targets, train_preds)
f1 = f1_score(train_targets, train_preds, average="macro")
auc = roc_auc_score(train_targets, train_probs)
logger.info(
f"Epoch {epoch+1} Train loss {train_loss/len(trainloader):.5}, Acc {acc*100:.3}%, F1 {f1*100:.3}%, AUC {auc*100:.4}%, grad {grad/len(trainloader)}"
)
net.eval()
test_loss = 0
test_targets = []
test_preds = []
test_probs = []
for batch in testloader:
img, label = batch
test_targets += label.numpy().tolist()
img, label = img.to(device), label.to(device)
with torch.no_grad():
pred = net(img)
loss = criterion(pred, label)
test_loss += loss.item()
test_preds += pred.cpu().detach().numpy().argmax(axis=1).tolist()
test_probs += pred.cpu().detach().numpy()[:, 1].tolist()
acc = accuracy_score(test_targets, test_preds)
f1 = f1_score(test_targets, test_preds, average="macro")
auc = roc_auc_score(test_targets, test_probs)
logger.info(
f"Epoch {epoch+1} Test loss {test_loss/len(testloader):.5}, Acc {acc*100:.3}%, F1 {f1*100:.3}%, AUC {auc*100:.4}%"
)
torch.save(net.state_dict, "net.pt")
def wallet_update_task():
em = ExchangeManager()
walletData = em.get_balance()
btcUsdValue = em.get_btc_usd_value()
totalBtcValue = 0.0
logger.info("Starting Wallet Update Task")
logger.debug("Checking Wallet Locks")
walletLockDeleteList = []
# Clear up the wallet locks.
for walletLockModel in DatabaseManager.get_all_wallet_trade_lock_models():
if DatabaseManager.get_coin_lock_model(walletLockModel.Ticker) == None:
walletLockDeleteList.append(walletLockModel.Ticker)
for walletLockTicker in walletLockDeleteList:
DatabaseManager.delete_wallet_trade_lock_model(walletLockTicker)
for key in DatabaseManager.get_all_supported_coin_models():
btcbalance = 0.0
usdBalance = 0.0
totalCoins = None
tickerModel = get_ticker(key)
try:
btcbalance = walletData[key.Ticker]['total'] * tickerModel.BTCVal
totalCoins = walletData[key.Ticker]['total']
usdBalance = btcUsdValue * btcbalance
except:
btcbalance = 0.0
totalCoins = 0.0
if key.Ticker == 'BTC':
btcbalance = walletData[key.Ticker]['total']
usdBalance = btcUsdValue * btcbalance
indexedCoin = DatabaseManager.get_index_coin_model(key.Ticker)
if indexedCoin is not None:
totalBtcValue = totalBtcValue + btcbalance
if DatabaseManager.create_coin_balance_model(key.Ticker, btcbalance,
usdBalance, totalCoins,
datetime.datetime.now()):
#logger.debug("Created Coin Balance Model - " + key.Ticker)
pass
else:
if DatabaseManager.update_coin_balance_model(
key.Ticker, btcbalance, btcUsdValue * btcbalance,
totalCoins, datetime.datetime.now()):
#logger.debug("Updated Coin Balance Model - " + key.Ticker)
pass
else:
logger.error("Failed Update Coin Balance Model - " +
key.Ticker)
totalUnrealizedGain = 0.0
totalRealizedGain = 0.0
for key in DatabaseManager.get_all_supported_coin_models():
tickerModel = get_ticker(key)
coinBalance = DatabaseManager.get_coin_balance_model(key.Ticker)
indexedCoin = DatabaseManager.get_index_coin_model(key.Ticker)
if indexedCoin is not None:
if DatabaseManager.update_index_coin_model(
indexedCoin.Ticker, indexedCoin.DesiredPercentage,
indexedCoin.get_distance_from_target(
coinBalance, totalBtcValue), indexedCoin.Locked):
logger.debug("Updated Indexed Coin Model - " +
indexedCoin.Ticker)
else:
logger.error("Failed To Update Indexed Coin Model - " +
indexedCoin.Ticker)
indexInfo = DatabaseManager.get_index_info_model()
if DatabaseManager.update_index_info_model(indexInfo.Active, totalBtcValue,
btcUsdValue * totalBtcValue,
indexInfo.BalanceThreshold,
indexInfo.OrderTimeout,
indexInfo.OrderRetryAmount,
indexInfo.RebalanceTickSetting):
logger.debug("Updated Index Info Model")
else:
logger.error("Failed To Update Index Info Model")
logger.info("Wallet Update Task Completed")
def GetBigInteger(self):
logger.info("Trying to get big integer %s " % self)
raise Exception("Not Supported")
def perform_buy_task(eligibleTicker, eligibleBuyAmount):
coinBuyIncomplete = True
coinBuyRetryCount = 0
buyOrderUUID = ""
indexInfo = DatabaseManager.get_index_info_model()
retryLimit = indexInfo.OrderRetryAmount
eligibleCoinTicker = DatabaseManager.get_ticker_model(eligibleTicker +
"/BTC")
em = ExchangeManager()
try:
partial_fill_amount = 0
partial_filled = False
DatabaseManager.create_coin_lock_model(eligibleTicker)
while coinBuyIncomplete:
if coinBuyRetryCount >= retryLimit:
coinBuyIncomplete = False
logger.info("Buying of coin " + eligibleTicker +
" failed after " + str(coinBuyRetryCount) +
" attempts")
break
# Cancel Order
else:
if CondexConfig.DEBUG == True:
logger.debug("Putting in buy order")
else:
logger.info("Buying %s of %s at %s", eligibleBuyAmount,
eligibleTicker, eligibleCoinTicker.BTCVal)
if partial_filled == True:
buyOrderUUID = em.create_buy_order(
eligibleTicker,
partial_fill_amount / eligibleCoinTicker.BTCVal,
eligibleCoinTicker.BTCVal)['id']
else:
buyOrderUUID = em.create_buy_order(
eligibleTicker, eligibleBuyAmount,
eligibleCoinTicker.BTCVal)['id']
time.sleep(60 * indexInfo.OrderTimeout)
# Check order succeded through
if CondexConfig.DEBUG == True:
logger.debug("Fetching order")
coinBuyIncomplete = False
else:
order_result = em.fetch_order(buyOrderUUID)
order_filled_amount = order_result['filled']
if order_result['status'] == "closed":
logger.info("Bought coin " + eligibleTicker + " for " +
str(order_result['price']))
coinBuyIncomplete = False
elif (
order_filled_amount * eligibleCoinTicker.BTCVal
) > CondexConfig.BITTREX_MIN_BTC_TRADE_AMOUNT and order_result[
'status'] == "open":
em.cancel_order(buyOrderUUID)
logger.debug("Bought partial of coin " +
eligibleCoinTicker + " for " +
str(order_result['price']))
coinBuyIncomplete = False
else:
coinBuyRetryCount = coinBuyRetryCount + 1
if CondexConfig.DEBUG == True:
logger.debug("Canceling buy order")
else:
try:
em.cancel_order(buyOrderUUID)
except:
coinBuyIncomplete = False
pass # order failed to cancel got filled previously
logger.debug("Buy Order Timeout Reached")
time.sleep(10) #Magic Number
except Exception as e:
logger.exception(e)
finally:
if CondexConfig.DEBUG != True:
DatabaseManager.delete_coin_lock_model(eligibleTicker)
def perform_sell_task(rebalanceTicker, rebalanceSellAmount):
coinSellIncomplete = True
coinSellRetryCount = 0
sellOrderUUID = ""
indexInfo = DatabaseManager.get_index_info_model()
retryLimit = indexInfo.OrderRetryAmount
eligibleCoinTicker = DatabaseManager.get_ticker_model(rebalanceTicker +
"/BTC")
em = ExchangeManager()
try:
partial_fill_amount = 0
partial_filled = False
DatabaseManager.create_coin_lock_model(rebalanceTicker)
while coinSellIncomplete:
if coinSellRetryCount >= retryLimit:
coinSellFailed = True
coinSellIncomplete = False
break
# Cancel Order
else:
rebalanceCoinTicker = DatabaseManager.get_ticker_model(
rebalanceTicker + "/BTC")
if CondexConfig.DEBUG == True:
logger.info("Placing Sell Order For " + rebalanceTicker +
"/BTC")
else:
logger.info("Selling " + str(rebalanceSellAmount) +
" of " + rebalanceTicker + " at " +
str(rebalanceCoinTicker.BTCVal))
sellOrderUUID = em.create_sell_order(
rebalanceTicker, rebalanceSellAmount,
rebalanceCoinTicker.BTCVal)['id']
time.sleep(60 * indexInfo.OrderTimeout)
# Check order succeded through
if CondexConfig.DEBUG == True:
logger.debug("Fetching order")
coinSellIncomplete = False
else:
order_result = em.fetch_order(sellOrderUUID)
order_filled_amount = order_result['filled']
if order_result['status'] == "closed":
logger.debug("Sold coin " + rebalanceTicker + " for " +
str(order_result['price']))
coinSellIncomplete = False
elif (
order_filled_amount * rebalanceCoinTicker.BTCVal
) > CondexConfig.BITTREX_MIN_BTC_TRADE_AMOUNT and order_result[
'status'] == "open":
em.cancel_order(sellOrderUUID)
logger.debug("Sold partial of coin " +
rebalanceTicker + " for " +
str(order_result['price']))
coinSellIncomplete = False
partial_filled = True
partial_fill_amount = order_filled_amount * rebalanceCoinTicker.BTCVal
else:
coinSellRetryCount = coinSellRetryCount + 1
if CondexConfig.DEBUG == True:
logger.debug("Canceling sell order")
else:
em.cancel_order(sellOrderUUID)
logger.debug("Sell Order Timeout Reached")
time.sleep(10) #Magic Number
except Exception as e:
logger.exception(e)
finally:
if CondexConfig.DEBUG != True:
DatabaseManager.delete_coin_lock_model(rebalanceTicker)
def main():
logger.info("starting...")
setup_signal_handling()
last_hello_emitted = time.time()
semaphore_file = settings.SEMAPHORE_FILE
if settings.USE_ECS_TASK_STRATEGY:
logger.info("using ECS Task strategy for semaphore token")
if len(settings.ECS_TASK_STRATEGY_ENDPOINT) is None:
logger.fatal("ECS_TASK_STRATEGY_ENDPOINT was empty")
return
r = requests.get(settings.ECS_TASK_STRATEGY_ENDPOINT)
metadata = r.json()
logger.debug("metadata was: " + json.dumps(metadata))
task_id = metadata["Labels"]["com.amazonaws.ecs.task-arn"].split("/")[1]
logger.debug("task_id: " + task_id)
semaphore_file = settings.ECS_TASK_STRATEGY_SEMAPHORE_FILE_TEMPLATE.replace("##task_id##", task_id)
logger.info("semaphore file set to " + semaphore_file)
if settings.USE_SEMAPHORE_FILE_STRATEGY:
if settings.SEMAPHORE_FILE_ENSURE_REMOVED:
logger.info("ensuring semaphore file at " + semaphore_file + " is removed")
if os.path.isfile(semaphore_file):
logger.info("semaphore file exists, removing")
os.unlink(semaphore_file)
while not requested_to_quit:
age = int(time.time() - last_hello_emitted)
if age > settings.SAY_HELLO_SECONDS:
logger.info("running...")
last_hello_emitted = time.time()
time.sleep(settings.SLEEP_SECONDS)
if requested_to_quit:
if settings.USE_SEMAPHORE_FILE_STRATEGY:
logger.info("touching semaphore file at " + semaphore_file)
open(semaphore_file, 'a').close()
logger.info("done")
def GetByteArray(self):
logger.info("Trying to get bytearray integer %s " % self)
raise Exception("Not supported")
