def update_best(self, state):
local_best = state.local_best
get_logger().debug(
f'At leaf, local best = {local_best}, global = {self.global_best}')
if self.global_best == None or local_best > self.global_best:
self.global_best = local_best
self.global_best_state = state
def get_next_states(self, visited_states=set()):
states = set()
for edge in self.free_edges.edges:
next_state = copy.deepcopy(self)
next_state.add_include_edge(*edge)
if next_state in visited_states:
continue
next_state.propagate_constraints(edge)
get_logger().debug(f'Added next state {next_state}')
states.add(next_state)
visited_states.add(next_state)
return states
def __init__(
self,
ids: Optional[list] = None,
bbox: Optional[BoundingBox] = None,
intersects: Optional[Geometry] = None,
contains: Optional[Geometry] = None,
time: Optional[str] = None,
limit: Optional[int] = None,
time_series: Optional[list] = None,
**kwargs,
): # pylint: disable=too-many-arguments
if bbox and intersects or intersects and contains or contains and bbox:
raise ValueError("""Only one of the following query parameters is
allowed at a query at any given time:
* bbox
* intersects
* contains.""")
self.__dict__.update(
kwargs
) # At the front so we do not accidentally overwrite some members
self.ids = ids
self.bbox = bbox
self.intersects = intersects
self.contains = contains
if not STACQuery.validate_datetime_str(time):
raise ValueError("""Time string could not be validated.
It must be one of the following formats:
<RFC3339> (for points in time)
<RFC3339>/<RFC3339> (for datetime ranges)""")
self.time = time
self.limit = 1 if limit is None else limit
if limit is not None and limit < 0:
helper.get_logger(__name__).warning(
"WARNING: limit parameter cannot be < 0, and has been automatically set to 1"
)
self.limit = 1
if time_series is not None:
for datestr in time_series:
if not STACQuery.validate_datetime_str(datestr):
raise ValueError(
"""Time string from time_series could not be validated.
It must be one of the following formats:
<RFC3339> (for points in time)
<RFC3339>/<RFC3339> (for datetime ranges)""")
self.time_series = time_series
def run(self):
logger = get_logger('SessionMgr')
threading_lock_interval_counter = 0
while True:
threading_lock_interval_counter += 1
time.sleep(self.polling_interval)
for k in self.data_set.keys():
v = self.data_set[k]
status = v["status"]
now = time.time()
if (now - v['timer']) > self.absolute_timeout:
try:
channel = v["channel"]
if channel in "cli snmp":
timestamp = time.strftime('%Y-%m-%d %H:%M:%S')
device_id = v["device_info"]["device_id"]
self.pedding_fuc(channel,device_id,now,timestamp)
del self.data_set[k]
logger.info('Task %s Timeout, cleared' % k)
except Exception,e:
logger.error(str(e))
elif "finish_timer" in v and (now - v['finish_timer']) > self.after_finish_timeout:
del self.data_set[k]
logger.info('Task %s Finished, cleared' % k)
def add_msg():
logger = get_logger('logs/add_msg.log')
logger.info("run add msg.")
# touser = "******"
touser = "******"
_send_hour = []
_send_day = 0
while True:
now = datetime.datetime.now()
day = now.day
hour = now.hour
if _send_day != day: _send_hour = []
if hour not in _send_hour:
_send_day = day
content = get_msg_content()
_send_hour.append(hour)
if content:
msg = {"to": touser, "msg": content}
logger.debug('msg: {0}'.format(content.encode('utf8')))
logger.info("will send '{0}' to '{1}'".format(
content.encode('utf8'), touser))
jobqe.add_job(settings.msg_queue_name, msg)
time.sleep(random.randint(1, 10) * 60)
def initialize():
"""
Setup neural networks and tensorboard logging.
"""
discriminator = Discriminator().to(config.device)
policy = generator.Policy().to(config.device)
rollout = generator.Policy().to(config.device)
discriminator.criterion = torch.nn.BCELoss()
discriminator.optimizer = torch.optim.Adam(discriminator.parameters(),
lr=config.d_learnrate)
policy.optimizer = torch.optim.Adam(policy.parameters(),
lr=config.g_learnrate)
hyperparameter = {
k: v
for k, v in config.__dict__.items() if not v == config.device
}
notes = ''''''
store.setup(loader.make_directory_with_timestamp(), hyperparameter, notes)
store.set('Policy Step', 0)
os.makedirs('{}/policies'.format(store.folder))
log = get_logger(store.folder, 'errors')
return discriminator, policy, rollout, log
def start_tests(self):
test_num = 1
for test in self.__test_list:
logger = get_logger(test_num)
list_flows = []
test_obj = Tests(self.__w3, logger, self.__inspector)
try:
func_args = test["args"]
if test_obj.is_thread(test["func"]):
list_flows = []
flows = test["flows"]
if flows > len(self.accounts):
flows = len(self.accounts)
for i in range(flows):
j = i + 1
if j == len(self.accounts):
j = 0
accounts = (self.accounts[i][0], self.accounts[i][1],
self.accounts[j][0], self.accounts[j][1])
func_args.insert(0, accounts)
list_flows.append(
Thread(target=test_obj.start_test,
args=(test["func"], func_args.copy())))
func_args.pop(0)
else:
list_flows.append(
Thread(target=test_obj.start_test,
args=(test["func"], func_args.copy())))
logger.info("Start test {0}(flows: {3}): {1}{2}".format(
test_num, test["func"], test["args"], len(list_flows)))
for flow in list_flows:
flow.start()
while True:
list_alive = [
state.is_alive() for state in list_flows
if state.is_alive()
]
if not list_alive:
break
except TypeError as e:
logger.error("\tUnhandled error in starting {2}:{0}{1}".format(
e.__class__.__name__, e, test["func"]))
continue
finally:
test_num += 1
sleep(
5
) # There are cases when transactions in same tests do not have time to process on the node
def find_best(self):
self.start_time = time.time()
pq = PriorityQueue()
pq.put((self.original_state.local_best, self.original_state))
while (not pq.empty()):
local_best, current_state = pq.get()
if self.is_visited(current_state):
continue
self.iter_count += 1
get_logger().debug(f'current_state: {current_state}')
if current_state.worst_than(self.global_best):
get_logger().debug(
f'current_state is worst than global best: {self.global_best}'
)
continue
next_states = current_state.get_next_states()
if len(next_states) == 0:
self.update_best(current_state)
for next_s in next_states:
pq.put((next_s.local_best, next_s))
self.complete_time = time.time() - self.start_time
return self.global_best_state
def __init__(self, params):
"""
Constructor for the main function. Loads data and creates computation graph.
Parameters
----------
params: Hyperparameters of the model
Returns
-------
"""
self.p = params
if not os.path.isdir(self.p.log_dir):
os.mkdir(self.p.log_dir)
if not os.path.isdir(self.p.emb_dir):
os.mkdir(self.p.emb_dir)
self.logger = hp.get_logger(self.p.name, self.p.log_dir,
self.p.config_dir)
self.logger.info(vars(self.p))
pprint(vars(self.p))
self.p.batch_size = self.p.batch_size
if self.p.l2 == 0.0:
self.regularizer = None
else:
self.regularizer = tf.contrib.layers.l2_regularizer(
scale=self.p.l2)
self.load_data()
self.add_placeholders()
nn_out = self.add_model()
self.loss = self.add_loss_op(nn_out)
if self.p.opt == 'adam':
self.train_op = self.add_optimizer(self.loss)
else:
self.train_op = self.add_optimizer(self.loss, isAdam=False)
self.merged_summ = tf.summary.merge_all()
def wechat():
logger = get_logger('logs/wechat.log')
import time
import itchat
from itchat.content import TEXT
@itchat.msg_register([TEXT])
def text_reply(msg):
logger.info('{0}: [{1}]{2}'.format(msg['FromUserName'], msg['Type'],
msg['Text'].encode('utf8')))
itchat.auto_login(enableCmdQR=True, hotReload=True)
itchat.run(blockThread=False)
logger.info("wechat login success.")
while True:
time.sleep(10)
x = jobqe.get_job(settings.msg_queue_name)
if not isinstance(x, dict): continue
data = x.get('data')
if not data or not isinstance(data, dict): continue
try:
msg = data.get('Data')
touser = msg.get('to')
content = msg.get('msg')
if not touser or not content: continue
touser = touser.strip()
us = itchat.search_friends(name=touser)
if len(us) == 0:
logger.info('no such user named: {0}'.format(touser))
continue
u = us[0]
itchat.send(content, toUserName=u["UserName"])
logger.info('send "{0}" to "{1}".'.format(content.encode('utf8'),
touser.encode('utf8')))
except:
logger.error(traceback.format_exc())
jobqe.ack_job(data.get('ID'))
parser.add_argument('--num_batches',
type=int,
default=10,
help='number of batches to evaluate')
opt = parser.parse_args()
print(opt)
# ----------
# Cuda or cpu
# ----------
cuda = is_cuda(opt.use_cpu)
# Logging init
if opt.logging:
print("Init logging...")
d_eval_logger_original = helper.get_logger(opt.log_port, 'd_eval_original')
d_eval_logger_cv2 = helper.get_logger(opt.log_port, 'd_eval_cv2')
d_eval_logger_completed = helper.get_logger(opt.log_port,
'd_eval_completed')
viz_image_logger = Visdom(port=opt.log_port, env="images")
# Loss function
criteria = torch.nn.BCELoss()
# Load generator and discriminator
generator = Generator(opt.batch_size, opt.latent_dim, opt.channels)
discriminator = Discriminator(opt.batch_size, opt.channels)
for model, model_name in [(generator, 'g'), (discriminator, 'd')]:
load_model(cuda, model, model_name)
if cuda:
# -*- coding=UTF-8 -*-
import argparse
import json
import re
from sqlalchemy import exc, func
from helper import get_logger, get_session, timeit
from models import FBShop, FBShopType
from fb_api import FBBot
__author__ = "Wen-Hao Lee"
__email__ = "*****@*****.**"
__copyright__ = "Copyright 2014, Numnum"
logger = get_logger('fb_fetcher', __file__)
class FBFetcher(object):
def __init__(self, env):
self.fb_bot = FBBot()
self.session = get_session(env)
@timeit(logger)
def fetch_new_shops(self, limit=None):
fb_shops = (self.session.query(FBShop).filter(
FBShop.name == None).order_by('update_ts').limit(limit).all())
processed = 0
for fb_shop in fb_shops:
processed += self._fetch_info(fb_shop)
logger.info("Try to fetch %d new fb_shops info, %d fetched" %
import json
import allParsers
import helper
logger = helper.get_logger()
def parseBlogs(domain, lock, search_results, filename, search_url):
logger.info('%s %s', search_results[0].link, filename)
sr_mapping = {result.link: result for result in search_results}
results = allParsers.mapper[domain].parse_all(sr_mapping.keys())
docs = {'docs':[]}
docs['search'] = search_url
for url in results:
search_result = sr_mapping[url]
try:
text = [content.text for content in results[url].content]
except Exception, e:
logger.error("Article Parsing Failed URL: %s %s", url, e)
text = ['Failure to Parse']
article = '\n'.join(text)
doc = {}
doc['id'] = hash(url)
doc['url'] = url
doc['title'] = search_result.title
doc['date'] = search_result.date
doc['article'] = article
doc['comment_url'] = results[url].comment_url
comments = []
for comment in results[url].comments:
help='if testing on cpu')
opt = parser.parse_args()
print(opt)
cuda = is_cuda(opt.use_cpu)
# Create z noise for generator input
def create_noise(batch_size, latent_dim):
return Variable(
Tensor(batch_size, latent_dim).normal_().view(-1, latent_dim, 1, 1))
# Logging
if opt.logging:
d_real_loss_logger = helper.get_logger(opt.log_port, 'd_loss_real')
d_fake_loss_logger = helper.get_logger(opt.log_port, 'd_loss_fake')
d_total_loss_logger = helper.get_logger(opt.log_port, 'd_loss_total')
g_loss_logger = helper.get_logger(opt.log_port, 'g_loss')
viz_image_logger = Visdom(port=opt.log_port, env="images")
# Loss function
criteria = torch.nn.BCELoss()
# Initialize generator and discriminator
generator = Generator(opt.batch_size, opt.latent_dim, opt.channels)
discriminator = Discriminator(opt.batch_size, opt.channels)
if cuda:
generator.cuda()
discriminator.cuda()
# -*- coding=UTF-8 -*-
import os
import json
import argparse
from sqlalchemy import exc
from helper import get_logger, base_path, get_session
from models import Track
__author__ = "Wen-Hao Lee"
__email__ = "*****@*****.**"
__copyright__ = "Copyright 2014, Numnum"
logger = get_logger('geo_walker', __file__)
class CityWalker(object):
rad = 500 # m
def __init__(self, area, env):
assert area in ['small', 'medium', 'large']
area_turns = {'small': 10, 'medium': 50, 'large': 120}
self.total_turns = area_turns[area]
self.session = get_session(env)
def load(self, file_name):
file_fullpath = os.path.join(base_path, 'data', file_name)
city_data = json.load(open(file_fullpath))
self.cities = city_data['cities']
def walk_all_cities(self):
from pathlib import Path
import shutil
from string import Template
import uuid
import copy
import xml.etree.ElementTree as Et
from geojson import FeatureCollection, Feature
import rasterio
from helper import (load_params, load_metadata,
ensure_data_directories_exist, save_metadata, get_logger,
read_write_bigtiff, SENTINEL1_L1C_GRD, SNAP_POLARIMETRIC)
from capabilities import set_capability
LOGGER = get_logger(__name__)
PARAMS_FILE = os.environ.get("PARAMS_FILE")
GPT_CMD = "{gpt_path} {graph_xml_path} -e {source_file}"
# pylint: disable=unnecessary-pass
class WrongPolarizationError(ValueError):
"""
This class passes to the next input file, if the current input file
does not include the polarization.
"""
pass
class SNAPPolarimetry:
"""
# -*- coding=UTF-8 -*-
import argparse
from sqlalchemy import exc
from helper import get_logger, get_session, timeit
from models import FBShop, GGShop
from gg_api import GGBot
__author__ = "Wen-Hao Lee"
__email__ = "*****@*****.**"
__copyright__ = "Copyright 2014, Numnum"
logger = get_logger('gg_matcher', __file__)
class GGMatcher(object):
def __init__(self, env, match_type='restaurant'):
self.gg_bot = GGBot()
self.session = get_session(env)
self.match_type = match_type
@timeit(logger)
def match_fb_shops(self, limit=None):
fb_shops = (
self.session.query(FBShop)
.filter(FBShop.name != None)
.filter(FBShop.ggid == None)
.order_by('update_ts').limit(limit).all())
processed = 0
for fb_shop in fb_shops:
processed += self._find_gg_shop(fb_shop)
'--debug',
help='enable debug output',
action='store_true')
parser.add_argument('-bd',
'--backend_debug',
help='enable debug output for the backend',
action='store_true')
parser.add_argument('-mf',
'--max_failures',
help='maximum amount of failures to allow',
type=int)
args = parser.parse_args()
# init logger
debug = args.debug
logger = get_logger("Config2Spec", 'DEBUG' if debug else 'INFO')
# name of the scenario
scenario = os.path.basename(args.scenario_path)
# all the necessary paths where config, fib and topology files are being stored
batfish_path = args.batfish_path # path to cloned Batfish repo directory
backend_path = args.backend_path # path to Batfish executable
scenario_path = args.scenario_path
batfish_port = args.port
# create backend manager
ms_manager = init_manager(backend_path, batfish_port)
# general settings
print traceback.format_exc()
self.write(
json.dumps(
dict(status='error',
message='Session error,please refresh')))
self.finish()
if __name__ == '__main__':
options.define("p", default=7777, help="run on the given port", type=int)
options.parse_command_line()
config = Configurations()
port = options.options.p
logger = get_logger("server", logging.DEBUG)
loop = ZMQIOLoop()
loop.install()
context = zmq.Context()
zmq_publish = context.socket(zmq.PUB)
zmq_publish.bind("tcp://127.0.0.1:%s" %
str(config.get_configuration("zmqPublish")))
zmq_dispatch = context.socket(zmq.REP)
zmq_dispatch.bind("tcp://127.0.0.1:%s" %
str(config.get_configuration("zmqDispatch")))
zmq_result = context.socket(zmq.PULL)
zmq_result.bind("tcp://127.0.0.1:%s" %
str(config.get_configuration("zmqResult")))
receiver = ZMQStream(zmq_result)
receiver.on_recv(on_worker_data_in)
import os
import sys
import time
import subprocess
import urllib2
from datetime import datetime
import argparse
import logging
import time
import socket
import math
import helper
time_diff = helper.time_diff
logger = helper.get_logger('exp_checkpoint')
class WorkerCommander:
def __init__(self, srv_ip, srv_port):
self.srv_ip = srv_ip
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.connect((srv_ip, srv_port))
def send_line(self, text):
self.s.send(text+'\n')
def recv_line(self):
return self.s.recv(1500)
def invoke_cmd(self, cmd):
import os
import requests
import time
import ConfigParser
import logging
import logging.config
from requests.exceptions import RequestException
from helper import get_logger, base_path
config = ConfigParser.ConfigParser()
config.read(os.path.join(base_path, 'config.ini'))
logger = get_logger('fb_api', __file__)
__author__ = "Wen-Hao Lee"
__email__ = "*****@*****.**"
__copyright__ = "Copyright 2014, Numnum"
class FBBot(object):
graph_url = "https://graph.facebook.com"
cooldown = 120 # sec
search_radius = 500 # m
def search_restaurant(self, lat, lon):
restaurants = self._search_place('restaurant', lat, lon)
steakhouses = self._search_place('steakhouse', lat, lon)
bars = self._search_place('bar', lat, lon)
return restaurants + steakhouses + bars
def _search_place(self, query, lat, lon):
parser.add_argument('--hub_ip', default='52.3.220.217')
parser.add_argument('--worker_ip', default='52.91.124.236')
parser.add_argument('--worker_port', default=1234, type=int)
# curl should support c-ares, which is need for --dns-servers option
parser.add_argument('--curl_bin_path', default='/Users/liang/.local/curl/bin/curl')
parser.add_argument('--curl_write_out', default='@/elasticity/scripts/curl-format-compact', help='control curl to display info. See curl --write-out')
parser.add_argument('--curl_output', default='curl.out')
parser.add_argument('-c', '--count', default=10, type=int, help='number of times to run the test')
parser.add_argument('--pico_id', default=-1, type=int)
parser.add_argument('--pico_state', default='hot', help='pick expected state of the pico from cold and hot')
parser.add_argument('--wait_time', default=1, type=int, help='sleep $wait_time seconds in each runs of non-hot pico experiments')
parser.add_argument('--set_mail_debug', default=False, action="store_true", help='use this flag to debug send mail')
parser.add_argument('--full_restore', default=False, action="store_true", help='download all pages, restore at once')
args = None #parser.parse_args()
logger = helper.get_logger()
def make_pico_uri(pico_id):
fmt = '{0:05}.pico'
return fmt.format(pico_id)
def ping(address):
count = 5
ret = subprocess.check_output(['ping', '-c', str(count), address])
lines = ret.splitlines()
for i in range(1, 1+count):
ping_line = 'PING: {0}'.format(lines[i])
logger.info(ping_line)
ping_stat_1 = 'PING-STAT-1: '+lines[-2]
# As of 2021 Feb 7, most of DMS is not supported as a Type for ASFF Resources
# https://docs.aws.amazon.com/securityhub/latest/userguide/securityhub-findings-format-attributes.html#asff-resources
WIKI = 'https://wiki.bnc.ca/x/g6SHM'
import json
import boto3
from helper import Evaluation, get_logger, adjust_tag_value
# The DMS Endpoint source engine does not have ssl_mode options/requirement
ENDPOINTS_SSLMODE_ENGINE_EXCEPTION = [
'Amazon S3', 'Amazon DynamoDB', 'Amazon Kinesis', 'Amazon Neptune',
'Amazon Redshift', 'Elasticsearch Service', 'Kafka'
]
log = get_logger()
def resource_types():
return (
'AWS::DMS::Endpoint',
'AWS::DMS::ReplicationTask',
'AWS::DMS::ReplicationInstance',
)
def evaluate_compliance_scheduled(event, credentials):
dms_client = boto3.client(
'dms',
aws_access_key_id=credentials['AccessKeyId'],
aws_secret_access_key=credentials['SecretAccessKey'],
# -*- coding=UTF-8 -*-
import argparse
from sqlalchemy import exc, or_, func
from helper import get_logger, get_session, timeit
from models import Track, FBShop
from fb_api import FBBot
__author__ = "Wen-Hao Lee"
__email__ = "*****@*****.**"
__copyright__ = "Copyright 2014, Numnum"
logger = get_logger('fb_searcher', __file__)
class FBSearcher(object):
def __init__(self, env):
self.fb_bot = FBBot()
self.session = get_session(env)
@timeit(logger)
def search_locations(self, limit=None):
locs = (
self.session.query(Track)
.order_by('update_ts').limit(limit).all())
updated = 0
for loc in locs:
updated += self.search_location(loc)
return updated
def search_location(self, loc):
import qiskit as q
from typing import List, Tuple
from qiskit.circuit.quantumcircuit import QuantumCircuit, QuantumRegister
from helper import grover
from helper import get_logger
from helper import measure_all
from helper import encode_graph
from helper import encode_value
from helper import simulate_circuit
from helper import initialize_circuit
from helper import symbolic_simulation
logging = get_logger(__name__)
def select_vertex(circuit : QuantumCircuit, registers : QuantumRegister, value : int, target : QuantumRegister, ancillas : QuantumRegister) -> QuantumCircuit:
circuit.barrier()
circuit = encode_value(circuit, registers[0], value, reverse=True)
controlled = [q for register in registers for q in register]
circuit.cnx(controlled, target, ancillas)
circuit = encode_value(circuit, registers[0], value, reverse=True)
return circuit
def find_edge(graph : List[Tuple[int, int, float]], current_mst : List[Tuple[int, int, float]], n_of_qbits_for_vertex : int = 3) -> Tuple[int, int]:
"""Quantum search algorithm to find an edge."""
# -*- coding=UTF-8 -*-
import argparse
import json
import re
from sqlalchemy import exc, func
from helper import get_logger, get_session, timeit
from models import FBShop, FBShopType
from fb_api import FBBot
__author__ = "Wen-Hao Lee"
__email__ = "*****@*****.**"
__copyright__ = "Copyright 2014, Numnum"
logger = get_logger('fb_fetcher', __file__)
class FBFetcher(object):
def __init__(self, env):
self.fb_bot = FBBot()
self.session = get_session(env)
@timeit(logger)
def fetch_new_shops(self, limit=None):
fb_shops = (
self.session.query(FBShop)
.filter(FBShop.name == None)
.order_by('update_ts').limit(limit).all())
processed = 0
for fb_shop in fb_shops:
# -*- coding=UTF-8 -*-
import os
import json
import argparse
from sqlalchemy import exc
from helper import get_logger, base_path, get_session
from models import Track
__author__ = "Wen-Hao Lee"
__email__ = "*****@*****.**"
__copyright__ = "Copyright 2014, Numnum"
logger = get_logger('geo_walker', __file__)
class CityWalker(object):
rad = 500 # m
def __init__(self, area, env):
assert area in ['small', 'medium', 'large']
area_turns = {
'small': 10,
'medium': 50,
'large': 120}
self.total_turns = area_turns[area]
self.session = get_session(env)
def load(self, file_name):
file_fullpath = os.path.join(base_path, 'data', file_name)
city_data = json.load(open(file_fullpath))
help='blend after completion?')
parser.add_argument('--num_batches',
type=int,
default=1,
help='number of batches to evaluate')
parser.add_argument('--mask_type', type=int, default=1, help='mask type')
opt = parser.parse_args()
print(opt)
cuda = is_cuda(opt.use_cpu)
# Logging
if opt.logging:
print("Init logging...")
contextual_loss_logger = helper.get_logger(opt.log_port, 'contextual_loss')
perceptual_loss_logger = helper.get_logger(opt.log_port, 'perceptual_loss')
completion_loss_logger = helper.get_logger(opt.log_port, 'completion_loss')
viz_image_logger = Visdom(port=opt.log_port, env="images")
# Loss function
criteria = torch.nn.BCELoss()
# Load generator and discriminator
generator = Generator(opt.batch_size, opt.latent_dim, opt.channels)
discriminator = Discriminator(opt.batch_size, opt.channels)
for model, model_name in [(generator, 'g'), (discriminator, 'd')]:
load_model(cuda, model, model_name)
if cuda:
generator.cuda()
def main(args):
# setup logging
log = get_logger(args.log)
log(args)
timestamp = datetime.now().strftime('%Y%m%d%H%M')
tb_writer = SummaryWriter("./output/{}/{}/{}/logs/".format(args.model, args.expname, args.dataset)\
+timestamp) if args.visual else None
config = getattr(configs, 'config_' + args.model)()
# instantiate the dmm
model = getattr(models, args.model)(config)
model = model.cuda()
if args.reload_from >= 0:
load_model(model, args.reload_from)
train_set = PolyphonicDataset(args.data_path + 'train.pkl')
valid_set = PolyphonicDataset(args.data_path + 'valid.pkl')
test_set = PolyphonicDataset(args.data_path + 'test.pkl')
#################
# TRAINING LOOP #
#################
times = [time.time()]
for epoch in range(config['epochs']):
train_loader = torch.utils.data.DataLoader(
dataset=train_set,
batch_size=config['batch_size'],
shuffle=True,
num_workers=1)
train_data_iter = iter(train_loader)
n_iters = train_data_iter.__len__()
epoch_nll = 0.0 # accumulator for our estimate of the negative log likelihood (or rather -elbo) for this epoch
i_batch = 1
n_slices = 0
loss_records = {}
while True:
try:
x, x_rev, x_lens = train_data_iter.next()
except StopIteration:
break # end of epoch
x, x_rev, x_lens = gVar(x), gVar(x_rev), gVar(x_lens)
if config['anneal_epochs'] > 0 and epoch < config[
'anneal_epochs']: # compute the KL annealing factor
min_af = config['min_anneal']
kl_anneal = min_af + (
1.0 - min_af) * (float(i_batch + epoch * n_iters + 1) /
float(config['anneal_epochs'] * n_iters))
else:
kl_anneal = 1.0 # by default the KL annealing factor is unity
loss_AE = model.train_AE(x, x_rev, x_lens, kl_anneal)
epoch_nll += loss_AE['train_loss_AE']
i_batch = i_batch + 1
n_slices = n_slices + x_lens.sum().item()
loss_records.update(loss_AE)
loss_records.update({'epo_nll': epoch_nll / n_slices})
times.append(time.time())
epoch_time = times[-1] - times[-2]
log("[Epoch %04d]\t\t(dt = %.3f sec)" % (epoch, epoch_time))
log(loss_records)
if args.visual:
for k, v in loss_records.items():
tb_writer.add_scalar(k, v, epoch)
# do evaluation on test and validation data and report results
if (epoch + 1) % args.test_freq == 0:
save_model(model, epoch)
test_loader = torch.utils.data.DataLoader(
dataset=test_set,
batch_size=config['batch_size'],
shuffle=False,
num_workers=1)
for x, x_rev, x_lens in test_loader:
x, x_rev, x_lens = gVar(x), gVar(x_rev), gVar(x_lens)
test_nll = model.valid(x, x_rev, x_lens) / x_lens.sum()
log("[val/test epoch %08d] %.8f" % (epoch, test_nll))
# -*- coding=UTF-8 -*-
import argparse
from sqlalchemy import exc
from helper import get_logger, get_session, timeit
from models import FBShop, GGShop
from gg_api import GGBot
__author__ = "Wen-Hao Lee"
__email__ = "*****@*****.**"
__copyright__ = "Copyright 2014, Numnum"
logger = get_logger('gg_matcher', __file__)
class GGMatcher(object):
def __init__(self, env, match_type='restaurant'):
self.gg_bot = GGBot()
self.session = get_session(env)
self.match_type = match_type
@timeit(logger)
def match_fb_shops(self, limit=None):
fb_shops = (self.session.query(FBShop).filter(
FBShop.name != None).filter(
FBShop.ggid == None).order_by('update_ts').limit(limit).all())
processed = 0
for fb_shop in fb_shops:
processed += self._find_gg_shop(fb_shop)
logger.info("Try to match %d gg_shops for fb_shops, %d matched" %
(len(fb_shops), processed))
# -*- coding=UTF-8 -*-
import argparse
import re
from sqlalchemy import exc, func
from helper import get_logger, get_session, timeit
from models import GGShop
from gg_api import GGBot
__author__ = "Wen-Hao Lee"
__email__ = "*****@*****.**"
__copyright__ = "Copyright 2014, Numnum"
logger = get_logger('gg_fetcher', __file__)
class GGFetcher(object):
def __init__(self, env):
self.gg_bot = GGBot()
self.session = get_session(env)
@timeit(logger)
def fetch_new_shops(self, limit=None):
gg_shops = (
self.session.query(GGShop)
.filter(GGShop.name == None)
.order_by('update_ts').limit(limit).all())
processed = 0
for gg_shop in gg_shops:
processed += self._fetch_info(gg_shop)
import boto3
from helper import ConfigItemEvaluation, get_logger, adjust_tag_value, InapplicableConfigItemEvaluation
WIKI = 'https://wiki.bnc.ca/x/dwQkNQ'
# Testing for wide Egress HTTPS is too flaky
# Thightly coupled with https://git.bnc.ca/projects/CLOUDSEC/repos/aws-security-group/browse/known_cidr_blocks.yaml?at=refs%2Fheads%2Fdevelop#740
# from netaddr import *
# IPSet(['0.0.0.0/0']) - IPSet(['10.0.0.0/8', '172.16.0.0/12', '192.168.0.0/16'])
# VALID_INTERNET_CIDR = ('0.0.0.0/5', '8.0.0.0/7', '11.0.0.0/8', '12.0.0.0/6', '16.0.0.0/4', '32.0.0.0/3', '64.0.0.0/2', '128.0.0.0/3', '160.0.0.0/5', '168.0.0.0/6', '172.0.0.0/12', '172.32.0.0/11', '172.64.0.0/10', '172.128.0.0/9', '173.0.0.0/8', '174.0.0.0/7', '176.0.0.0/4', '192.0.0.0/9', '192.128.0.0/11', '192.160.0.0/13', '192.169.0.0/16', '192.170.0.0/15', '192.172.0.0/14', '192.176.0.0/12', '192.192.0.0/10', '193.0.0.0/8', '194.0.0.0/7', '196.0.0.0/6', '200.0.0.0/5', '208.0.0.0/4', '224.0.0.0/3')
# TODO The following check is not implemented yet:
# - Has to be attached to only one type of resource
LOG = get_logger()
def evaluate_compliance_item_Change(event, configuration_item):
"""TODO This control is not enabled yet"""
return [InapplicableConfigItemEvaluation(configuration_item, event)]
securityhub_resource_details = {
'GroupName': configuration_item['configuration']['groupName'],
'GroupId': configuration_item['configuration']['groupId'],
'OwnerId': configuration_item['configuration']['ownerId'],
'VpcId': configuration_item['configuration']['vpcId'],
# TODO IpPermissions keys start with uppercase
# https://docs.aws.amazon.com/securityhub/latest/userguide/securityhub-findings-format-attributes.html#asff-resourcedetails-awsec2securitygroup
# 'IpPermissions': configuration_item['configuration']['ipPermissions'],
# 'IpPermissionsEgress': configuration_item['configuration']['ipPermissionsEgress']
}
eval = ConfigItemEvaluation(__name__, configuration_item, WIKI, event, 'AwsEc2SecurityGroup', securityhub_resource_details)
import pandas as pd
import argparse
import pickle
from gensim.summarization import bm25
# Custom functions
import sys
sys.path.append('./src')
import helper as he
import prepare as pr
import custom as cu
import data as dt
logger = he.get_logger(location=__name__)
rank_type_lookup = {
'historical': 0, #NOTE: currently only 0 is supported
'textblocks': 1,
'historical_thread': 2
}
class Rank():
def __init__(self, task, rank_type='historical', inference=False):
self.dt_rank = dt.Data(task=task, inference=inference)
# Load bm25
with open(self.dt_rank.get_path('fn_rank', dir='model_dir'),
'rb') as fh:
self.bm = pickle.load(fh)
self.data = pickle.load(fh)
# -*- coding: utf-8; -*-
import os
import time
import requests
import ConfigParser
from requests.exceptions import RequestException
from helper import get_logger, base_path
__author__ = "Wen-Hao Lee"
__email__ = "*****@*****.**"
__copyright__ = "Copyright 2014, Numnum"
config = ConfigParser.ConfigParser()
config.read(os.path.join(base_path, 'config.ini'))
logger = get_logger('gg_api', __file__)
class GGBot(object):
map_url = "https://maps.googleapis.com/maps/api/place"
cooldown = 180 # sec
search_radius = 1200 # m
def search_nearby(self, keyword, type_, lat, lon):
nearby_url = "%s/%s" % (self.map_url, "nearbysearch/json")
params = {
'keyword': keyword,
'location': "%s,%s" % (lat, lon),
'radius': self.search_radius,
'language': "zh-TW",
'type': type_,
