def __init__(self):
self.logger = log.SysLogger().log
self.platform = Config().get_config('wyt')['platform']
self.app_key = Config().get_config('wyt')['app_key']
self.token = Config().get_config('wyt')['token']
self.client_id = Config().get_config('wyt')['client_id']
self.client_secret = Config().get_config('wyt')['client_secret']
def __init__(self):
super().__init__()
self.base_url = "http://openapi.winit.com.cn/openapi/service"
self.app_key = Config().get_config('gucang')['app_key']
self.token = Config().get_config('gucang')['token']
self.base_name = 'mssql'
self.cur = self.base_dao.get_cur(self.base_name)
self.con = self.base_dao.get_connection(self.base_name)
def __init__(self):
super().__init__()
self.base_url = "https://oms.goodcang.net/default/svc/web-service"
self.app_key = Config().get_config('gucang')['app_key']
self.token = Config().get_config('gucang')['token']
self.base_name = 'mssql'
self.cur = self.base_dao.get_cur(self.base_name)
self.con = self.base_dao.get_connection(self.base_name)
def __init__(self, account):
self.logger = log.SysLogger().log
self.api_name = Config().get_config('ali')['api_name']
self.app_key = Config().get_config('ali')['app_key']
self.app_secret_key = Config().get_config('ali')['app_secret_key']
self.refresh_token = Config().get_config(
'ali')['refresh_token'][account]
self.token = self._get_access_token()
def __init__(self):
super().__init__()
self.config = Config().get_config('ebay.yaml')
self.mongo = MongoClient('192.168.0.150', 27017)
self.mongodb = self.mongo['operation']
self.col = self.mongodb['ebay_description_template']
self.col1 = self.mongodb['ebay_description_group']
def make_config(self,
z_dim=256,
rec_x_weight=300,
evaluation_verbose=False):
config = {
"privater": {
"type": "vae",
"z_dim": z_dim,
"rec_x_weight": rec_x_weight,
"encrypt_with_noise": True,
"optimizer": {
"type": "adam",
"lr": 0.0003,
}
},
"dataset": {
"type": "ferg"
},
"trainer": {
"type": "keras",
"epochs": 50,
"save_model": True
},
"evaluaters": [{
"type": "utility",
"z_dim": z_dim,
"verbose": evaluation_verbose
}, {
"type": "private",
"z_dim": z_dim,
"verbose": evaluation_verbose
}]
}
return Config(config)
def make_config(self, NAME):
config = {
"privater": {
"type": "cvae_mi",
"z_dim": z_dim,
"global_weight": 1,
"rec_x_weight": 10,
"local_weight": 1,
"encrypt_with_noise": True,
"optimizer": {
"type": "adam",
"lr": 0.0003,
}
},
"dataset": {
"type": "ferg"
},
"trainer": {
"type": "keras",
"epochs": 100
},
"evaluaters": [{
"type": "private",
"z_dim": z_dim,
"verbose": evaluation_verbose
}, {
"type": "reconstruction",
"base_dir": NAME
}]
}
return Config(config)
def __init__(self):
super().__init__()
self.config = Config().get_config('ebay.yaml')
self.task = self.get_mongo_collection('operation',
'wish_off_shelf_task')
self.product_list = self.get_mongo_collection('operation',
'wish_products')
def __init__(self):
super().__init__()
self.config = Config().get_config('ebay.yaml')
self.batch_id = str(datetime.datetime.now() - datetime.timedelta(days=7))[:10]
self.base_name = 'mssql'
self.cur = self.base_dao.get_cur(self.base_name)
self.con = self.base_dao.get_connection(self.base_name)
def __init__(self):
super().__init__()
self.config = Config().get_config('ebay.yaml')
self.base_name = 'mssql'
self.cur = self.base_dao.get_cur(self.base_name)
self.con = self.base_dao.get_connection(self.base_name)
self.col = self.get_mongo_collection('operation', 'ebay_product_list')
def __init__(self):
super().__init__()
self.config = Config().get_config('ebay.yaml')
self.batch_id = '2020-08-01'
self.base_name = 'mssql'
self.cur = self.base_dao.get_cur(self.base_name)
self.con = self.base_dao.get_connection(self.base_name)
def main():
config = Config()
config.hidden_layer = 100
config.discount = 0.99
config.use_gae = True
config.gae_tau = 0.95
config.gradient_clip = 0.5
config.rollout_length = 2048
config.optimization_epochs = 10
config.mini_batch_size = 64
config.ppo_ratio_clip = 0.2
config.entropy_weight = 0.01
env = gym.make('mo')
max_steps = env.spec.timestep_limit
print(max_steps)
obs_size = np.shape(env.observation_space)[0]
action_size = np.shape(env.action_space)[0]
print(obs_size, action_size)
agent = PPOAgent(config, obs_size, action_size, env)
for i in range(10000):
print('iter', i)
states, actions, log_probs_old, returns, advantages = agent.sample()
agent.ppo_update(states, actions, log_probs_old, returns, advantages)
def make_config(self, NAME):
config = {
"privater": {
"type": "ad_vae",
"z_dim": z_dim,
"rec_x_weight": 64 * 64 * 3,
"prior_weight": 1,
"encrypt_with_noise": True,
"optimizer": {
"type": "adam",
"lr": 0.0003,
}
},
"dataset": {
"type": "ferg"
},
"trainer": {
"type": "adv",
"d_iter": 2,
"epochs": 100
},
"evaluaters": [{
"type": "utility",
"z_dim": z_dim,
"verbose": evaluation_verbose
}, {
"type": "private",
"z_dim": z_dim,
"verbose": evaluation_verbose
}, {
"type": "reconstruction",
"base_dir": NAME
}]
}
return Config(config)
def login_session():
base_url = 'http://139.196.109.214/index.php/myibay/login/redirect/%252Findex.php%252Fmyibay'
config = Config()
payload = config.get_config(f'ibay_user_info')
session = requests.Session()
session.post(base_url, data=payload)
return session
def load_config(dataset_name):
cfg = Config()
''' Experiment '''
cfg.experiment_idx = 1
cfg.trial_id = None
cfg.train_mode = 'train'
''' Dataset '''
cfg.dataset_name = dataset_name
cfg.set_hint_patch_shape((96, 96, 96))
cfg.num_classes = 4
''' Model '''
cfg.model_name = 'unet'
cfg.first_layer_channels = 32
cfg.num_input_channel = 1
cfg.step_count = 4
''' Training '''
cfg.numb_of_epochs = 25000
cfg.eval_every = 1
cfg.lamda_ce = 1
cfg.batch_size = 1
cfg.learning_rate = 1e-4
''' Priors '''
cfg.priors = None
cfg.augmentation_shift_range = 15
''' Save at '''
cfg.save_path = '/cvlabdata1/cvlab/datasets_udaranga/experiments/miccai2019/'
cfg.save_dir_prefix = 'Experiment_'
return cfg
def __init__(self):
print('Iniciando Classifier')
self.vocabulary = []
self.modules = {}
self.input_vector = []
self.output_vector = []
self.config = Config()
self.translator = YandexTranslate(self.config.get_token_yandex())
self.dao = Mongo_DAO('localhost', 27017, 'classifier')
def __init__(self,
model_name,
use_bert,
bert_type=1,
max_len_bert=None,
bert_trainable=False,
bert_config_file=None,
bert_model_file=None,
feature=False,
swa=True,
seed=42,
columns='title',
computers=False):
self.config = Config()
self.model_name = model_name
self.use_bert = use_bert
self.bert_type = bert_type
self.bert_trainable = bert_trainable
self.feature = feature
self.store_name = model_name
if self.use_bert:
if max_len_bert > 100:
self.config.batch_size = 16
else:
self.config.batch_size = 32
self.store_name += '_bert%d' % self.bert_type
if not bert_trainable:
self.store_name += '_fix'
if self.feature:
self.store_name += '_f'
if computers:
self.store_name += '_computers'
self.seed = seed
if isinstance(columns, list):
columns = '_'.join(columns)
self.columns = columns
self.config.checkpoint_dir = os.path.join(self.config.checkpoint_dir,
columns)
if not os.path.exists(self.config.checkpoint_dir):
os.makedirs(self.config.checkpoint_dir)
self.store_name = os.path.join(self.config.checkpoint_dir,
self.store_name)
print(self.store_name)
if not os.path.exists(self.store_name):
os.mkdir(self.store_name)
self.config.max_len_bert = max_len_bert
if bert_trainable:
self.optimizer = Adam(lr=2e-5)
else:
self.optimizer = 'adam'
self.bert_config_file = bert_config_file
self.bert_model_file = bert_model_file
self.callbacks = []
self.swa = swa
def __init__(self):
super().__init__()
self.config = Config().get_config('ebay.yaml')
self.base_name = 'mssql'
self.today = datetime.datetime.today() - datetime.timedelta(hours=8)
self.log_type = {1: "刊登商品", 2: "添加多属性"}
self.cur = self.base_dao.get_cur(self.base_name)
self.con = self.base_dao.get_connection(self.base_name)
self.tokens = self.get_tokens()
def __init__(self, rule_id=None):
super().__init__()
self.rule_id = rule_id
self.headers = headers
config = Config()
self.haiying_info = config.get_config('haiying')
self.mongo = MongoClient('192.168.0.150', 27017)
self.mongo = motor.motor_asyncio.AsyncIOMotorClient('192.168.0.150', 27017)
self.mongodb = self.mongo['product_engine']
def __init__(self):
super().__init__()
self.config = Config().get_config('ebay.yaml')
self.base_name = 'mssql'
self.warehouse = 'mysql'
self.cur = self.base_dao.get_cur(self.base_name)
self.con = self.base_dao.get_connection(self.base_name)
self.warehouse_cur = self.base_dao.get_cur(self.warehouse)
self.warehouse_con = self.base_dao.get_connection(self.warehouse)
def train_parse_fn(example):
"""
:param example: 序列化的输入
:return:
"""
config = Config()
features = tf.parse_single_example(serialized=example,
features={
'img_name':
tf.FixedLenFeature([], tf.string),
'img_height':
tf.FixedLenFeature([], tf.int64),
'img_width':
tf.FixedLenFeature([], tf.int64),
'img':
tf.FixedLenFeature([], tf.string),
'gtboxes_and_label':
tf.FixedLenFeature([], tf.string)
})
img_name = features['img_name']
img_height = tf.cast(features['img_height'], tf.int32)
img_width = tf.cast(features['img_width'], tf.int32)
img = tf.decode_raw(features['img'], tf.uint8)
img = tf.reshape(img, shape=[img_height, img_width, 3])
img = tf.cast(img, tf.float32)
gt_boxes_and_label = tf.decode_raw(features['gtboxes_and_label'], tf.int32)
gt_boxes_and_label = tf.reshape(gt_boxes_and_label, [-1, 5])
# shape of img is (1024, 1024, 3), image_window(4,)[y1, x1, y2, x2]
img, gt_boxes_and_label, image_window = image_preprocess.image_resize_pad(
img_tensor=img,
gtboxes_and_label=gt_boxes_and_label,
target_side=config.TARGET_SIDE)
img, gt_boxes_and_label = image_preprocess.random_flip_left_right(
img_tensor=img, gtboxes_and_label=gt_boxes_and_label)
# choose or padding make the gt_bbox_labels is FAST_RCNN_MAX_INSTANCES
num_objects = tf.shape(gt_boxes_and_label)[0]
object_index = tf.range(num_objects)
object_index = tf.random_shuffle(object_index)
object_index = object_index[:config.FAST_RCNN_MAX_INSTANCES]
gt_boxes_and_label = tf.gather(gt_boxes_and_label, object_index)
anchor = make_anchor.generate_pyramid_anchors(config)
minibatch_indices, minibatch_encode_gtboxes, \
rpn_objects_one_hot = boxes_utils.build_rpn_target(gt_boxes_and_label[:, :4], anchor, config)
num_padding = config.FAST_RCNN_MAX_INSTANCES - tf.shape(
gt_boxes_and_label)[0]
# (FAST_RCNN_MAX_INSTANCES, 5)[y1, x1, y2, x2, label]
num_padding = tf.maximum(num_padding, 0)
gt_box_label_padding = tf.zeros((num_padding, 5), dtype=tf.int32)
gt_boxes_and_label = tf.concat([gt_boxes_and_label, gt_box_label_padding],
axis=0)
return {"image_name": img_name, "image": img, "image_window": image_window}, \
{"gt_box_labels": gt_boxes_and_label, "minibatch_indices": minibatch_indices,
"minibatch_encode_gtboxes": minibatch_encode_gtboxes, "minibatch_objects_one_hot": rpn_objects_one_hot}
def __init__(self):
super().__init__()
config = Config().config
self.token = config['ur_center']['token']
self.base_name = 'mssql'
self.warehouse = 'mysql'
self.cur = self.base_dao.get_cur(self.base_name)
self.con = self.base_dao.get_connection(self.base_name)
self.warehouse_cur = self.base_dao.get_cur(self.warehouse)
self.warehouse_con = self.base_dao.get_connection(self.warehouse)
def __init__(self,tupianku_name=1):
super().__init__()
config = Config()
self.tupianku_name = tupianku_name
self.tupianku_info = config.get_config(f'tupianku{tupianku_name}')
# self.proxy_url = "http://127.0.0.1:1080"
self.proxy_url = None
self.session = aiohttp.ClientSession()
self.base_name = 'mssql'
self.cur = self.base_dao.get_cur(self.base_name)
self.con = self.base_dao.get_connection(self.base_name)
def __init__(self):
super().__init__()
self.config = Config().get_config('ebay.yaml')
self.base_name = 'mssql'
self.today = datetime.datetime.today() - datetime.timedelta(hours=8)
self.log_type = {1: "刊登商品", 2: "添加多属性"}
self.cur = self.base_dao.get_cur(self.base_name)
self.con = self.base_dao.get_connection(self.base_name)
self.tokens = self.get_tokens()
self.col_temp = self.get_mongo_collection('operation', 'ebay_template')
self.col_task = self.get_mongo_collection('operation', 'ebay_task')
self.col_log = self.get_mongo_collection('operation', 'ebay_log')
def __init__(self, temp_dir, logs_dir=None):
if not logs_dir:
logger.debug("looking for logs")
logs_dir = Config().run_folder
self.logs_dir = logs_dir
self.temp_dir = temp_dir
self.last_log = self._get_last_log()
self._make_log_copy()
self.log_file = os.path.join(self.temp_dir,
os.path.basename(self.last_log))
self.full_text = self._get_log_content()
def __init__(self):
super().__init__()
config = Config().config
self.token = config['ur_center']['token']
self.op_token = config['op_center']['token']
self.base_name = 'mssql'
self.warehouse = 'mysql'
self.cur = self.base_dao.get_cur(self.base_name)
self.con = self.base_dao.get_connection(self.base_name)
self.warehouse_cur = self.base_dao.get_cur(self.warehouse)
self.warehouse_con = self.base_dao.get_connection(self.warehouse)
self.col = self.get_mongo_collection('operation', 'wish_template')
def process(**opts):
"""
process method validates inputs and executes all workflow methods for one provider
:param opts: user/commandline inputs
:return: None
"""
session = None
try:
configs = Config()
validate_n_format(configs, **opts)
opts['provider'] = opts['provider'].lower()
provider = opts['provider'].lower()
# Start file download processing
downloader = _get_downloader_obj(provider)
auth_config = configs.auth_config[provider]
access_config = configs.access_config[provider]
opts['access_urls'] = access_config['access-url'].copy()
logging.info(f"Retrieval initiated for {provider}")
if auth_config:
# Authenticate and login
session, response_dict = downloader.authenticate(provider)
opts['response_dict'] = response_dict
# Access
downloader.access(session, **opts)
# Parse
downloader.parse(**opts)
# Filter files for download
downloader.filter(**opts)
# Download files
downloader.download_files(session, **opts)
else:
logging.debug(
f'Authentication not required for provider :: {provider}')
# For 'FM' provider, access, parse and filter covered in parse method
downloader.parse(**opts)
# Download files
downloader.download_files(session, **opts)
# Transfer files to desired path
downloader.file_transfer(**opts)
logging.info(f"Storage complete for {provider}")
except DownloadException as u:
u.log_message()
raise u
except Exception as e:
error_logger.exception(f'0000: ERROR - Unknown exception :: {e}')
raise e
finally:
# Close session after file download
if session:
session.close()
return opts
def main():
parser = argparse.ArgumentParser()
parser.add_argument('config_path')
args = parser.parse_args()
# get config
with open(args.config_path, 'r') as f:
cfg = json.load(f)
config = Config(**cfg)
###################
# set paths
paths_images_train = config.train_data_refined_dir_ims.split(',')
print("00_gen_folds.py: path_images_train:", paths_images_train)
train_files = []
for p in paths_images_train:
train_files.extend(os.listdir(p))
print("train_files[:10]:", train_files[:10])
weight_save_path = os.path.join(config.path_results_root, 'weights',
config.save_weights_dir)
os.makedirs(weight_save_path, exist_ok=True)
folds_save_path = os.path.join(weight_save_path, config.folds_file_name)
if os.path.exists(folds_save_path):
print("folds csv already exists:", folds_save_path)
return
else:
print("folds_save_path:", folds_save_path)
shuffle(train_files)
s = {k.split('_')[0] for k in train_files}
d = {k: [v for v in train_files] for k in s}
folds = {}
if config.num_folds == 1:
nfolds = int(np.rint(1. / config.default_val_perc))
else:
nfolds = config.num_folds
idx = 0
for v in d.values():
for val in v:
folds[val] = idx % nfolds
idx += 1
df = pd.Series(folds, name='fold')
df.to_csv(folds_save_path, header=['fold'], index=True)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('config_path')
args = parser.parse_args()
with open(args.config_path, 'r') as f:
cfg = json.load(f)
config = Config(**cfg)
root_dir = os.path.join(config.path_results_root, config.test_results_dir)
# # if not using config
# parser.add_argument('--root_dir', default='', type=str,
# help='Root directory of data')
# args = parser.parse_args()
#
# weight_keys = ['length', 'travel_time_s']
# # weight_keys = ['length', 'travel_time_s']
# verbose = True
# root_dir = args.root_dir
# # Debug?
# t0 = time.time()
# weight_keys = ['length', 'travel_time_s', 'inferred_speed_mph']
# verbose = False #True
# pkl_dir = os.path.join(root_dir, 'graphs_speed')
# output_csv_path = os.path.join(root_dir,
# 'solution_init_debug.csv')
# df = pkl_dir_to_wkt(pkl_dir,
# output_csv_path=output_csv_path,
# weight_keys=weight_keys, verbose=verbose)
# tf = time.time()
# print("Submission file (debug):", output_csv_path)
# print("Time to create init submission:", tf-t0, "seconds")
# Final
t0 = time.time()
weight_keys = ['length', 'travel_time_s']
verbose = False #True
pkl_dir = os.path.join(root_dir, 'graphs_speed')
output_csv_path = os.path.join(root_dir, 'solution.csv')
df = pkl_dir_to_wkt(pkl_dir,
output_csv_path=output_csv_path,
weight_keys=weight_keys,
verbose=verbose)
tf = time.time()
print("Submission file:", output_csv_path)
print("Time to create submission:", tf - t0, "seconds")
def __init__(self):
super().__init__()
self.config = Config().get_config('ebay.yaml')
self.base_name = 'mssql'
self.warehouse = 'mysql'
self.cur = self.base_dao.get_cur(self.base_name)
self.con = self.base_dao.get_connection(self.base_name)
self.warehouse_cur = self.base_dao.get_cur(self.warehouse)
self.warehouse_con = self.base_dao.get_connection(self.warehouse)
self.base_url = "http://openapi.winit.com.cn/openapi/service"
self.end_time = str(datetime.datetime.today() -
datetime.timedelta(days=1))[:10]
self.begin_time = str(datetime.datetime.today() -
datetime.timedelta(days=91))[:10]
self.oauth = wytOauth.Wyt()
