class ImportCSV(SparkOperator):
'''import csv formate data'''
OP_NAME = 'import-csv'
OP_CATEGORY = 'data-import'
def __init__(self):
super(ImportCSV, self).__init__()
self.op_input_num = 0
self.op_output_num = 1
self.op_status = OperatorStatus.INIT
self.op_script_location = 'resources/spark_operators/data_import/import_csv.py'
self.op_backend = 'spark'
self.input_path = None
self.delimiter = None
def init_operator(self, op_json_param):
self.op_json_param = op_json_param
self.op_running_mode = self.op_json_param['running-mode'] if 'running-mode' in self.op_json_param else 'script'
self.op_local = bool(self.op_json_param['local']) if 'local' in self.op_json_param else True
if self.op_local:
self.op_script_location = os.getcwd() + '/' + self.op_script_location
self.op_working_directory = self.op_json_param['op-working-directory'] if 'op-working-directory' in self.op_json_param else None
self.op_logger = Logger(self.op_working_directory + '/log/import-csv_' + str(self.op_json_param['op-index']))
self.input_path = self.op_json_param['input-path']
self.delimiter = self.op_json_param['delimiter'] if 'delimiter' in self.op_json_param else ','
def run_function_mode(self):
return self.op_status
def run_script_mode(self):
run_command = 'spark-submit --master '
if self.op_local:
run_command = run_command + 'local[2] '
self.op_result.append(self.op_working_directory + 'output/' + self.op_json_param['op-index'] + '-output')
run_command = run_command + self.op_script_location + ' ' + self.input_path + ' ' + self.op_result[0] + ' ' + self.delimiter
sub_proc = subprocess.Popen(run_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
for line in iter(sub_proc.stdout.readline, b''):
self.op_logger.info(line)
sub_proc.stdout.close()
sub_proc.wait()
self.op_status = sub_proc.returncode
return self.op_status
def azkaban_script(self):
run_command = 'spark-submit --master '
if self.op_local:
run_command = run_command + 'local[2] '
self.op_result.append(self.op_working_directory + 'output/' + self.op_json_param['op-index'] + '-output')
run_command = run_command + self.op_script_location + ' ' + self.input_path + ' ' + self.op_result[0] + ' ' + self.delimiter
return run_command
class TestNotifier(object):
def __init__(self):
self.service_name = 'test_service'
self.meta = meta
self.request_body = {
'jobId': 1,
'stepId': 1,
'batchId': 0,
'retry': 0,
'status': 3,
'message': 'RUNNING',
'groupName': 'TestGroupName'
}
self.is_notifiable = True
#self.is_notifiable = False
self.logger = Logger(log_level="info",
vendor_key=20,
retailer_key=100)
self.logger.info(
'tttttttttttttttteeeeeeeeeeeeeeeeeeessssssssssstttttttttttttt')
@Notifier
def test(self):
print('yoyo')
@Notifier
def test_failure(self):
raise RuntimeError('Calling error.')
def main(self):
self.test()
#self.test_failure()
print('done')
class BaiDuGet(unittest.TestCase):
'''百度get测试，带headers'''
def setUp(self):
self.baidu = Baidu("case.xls", "baidu")
self.logger = Logger("FOX",
cmd_log_level=logging.INFO,
file_log_level=logging.INFO)
def test_baidu_get(self):
try:
status_code = self.baidu.baidu_get()
self.assertEqual(status_code, 200)
self.logger.info("baidu get success")
except BaseException as e:
self.logger.error("baidu get fail")
raise
def tearDown(self):
pass
class AppEE(object):
def __init__(self,
service=None,
endpoint=None,
api_key=None,
export=EXPORT_CLIPBOARD):
self.export = export
self.api_request = ApiRequest(service=service,
endpoint=endpoint,
api_key=api_key)
self.logger = Logger().get_logger()
def __export_app(self, app_uuid, text, target):
try:
if self.export == EXPORT_CLIPBOARD:
pyperclip.copy(text)
self.logger.info(
'Your generated script is copied to clipboard!')
# mb.showinfo('Info', 'Your generated script is copied to clipboard!')
elif self.export == EXPORT_FILE:
file_path = ''
if target == TARGET_INCREMENTAL:
file_path = os.path.join(
os.getcwd(), f'update_app_parameters_{app_uuid}.sql')
elif target == TARGET_FINAL:
file_path = os.path.join(os.getcwd(),
f'final_app_{app_uuid}.sql')
else:
raise Exception(
f"The file path '{file_path}' does not exist.")
with open(file_path, 'w') as f:
f.write(text)
self.logger.info(
f'Your generated script is written to the file {file_path}.'
)
# mb.showinfo('Info', f'Your generated script is written to the file {file_path}.')
except Exception as e:
self.logger.exception(f'Failed to export your script.', exc_info=e)
# mb.showerror('Error', f'Failed to export your script. {repr(e)}')
def __export_all_apps(self, text):
try:
if self.export == EXPORT_CLIPBOARD:
pyperclip.copy(text)
self.logger.info(
'Your generated script is copied to clipboard!')
elif self.export == EXPORT_FILE:
file_path = 'insert_all_apps.sql'
with open(file_path, 'w') as f:
f.write(text)
self.logger.info(
f'Your generated script is written to the file {file_path}.'
)
except Exception as e:
self.logger.exception(f'Failed to export your script.', exc_info=e)
def get_app(self, uuid):
path = f'timeSeriesGroups/apps/{uuid}'
return self.api_request.get(path=path)
def get_all(self):
path = 'timeSeriesGroups/apps/all'
return self.api_request.get(path=path)
def is_super_user(self):
path = 'superusers/check'
res = self.api_request.get(path=path)
return res['value']
def sql_update_parameters(self, app_uuid):
try:
app = self.get_app(uuid=app_uuid)
parameters = json.dumps(app['parameters']).replace("'", "''")
res = f"UPDATE time_series_group_app SET parameters = '{parameters}'::jsonb WHERE time_series_group_app_uuid = '{app_uuid}'::uuid;"
self.logger.info('----- UPDATE APP PARAMETERS STATEMENT -----')
self.logger.info(res)
self.__export_app(app_uuid, res, TARGET_INCREMENTAL)
except Exception as e:
self.logger.exception(f'Failed to get app {app_uuid}.', exc_info=e)
def sql_insert_final(self, app_uuid):
try:
app = self.get_app(uuid=app_uuid)
settings = {}
if 'description' in app:
settings['description'] = app['description']
if 'show_result' in app:
settings['show_result'] = app['show_result']
if 'alertable' in app:
settings['alertable'] = app['alertable']
if 'trainable' in app:
settings['trainable'] = app['trainable']
if 'inferenceable' in app:
settings['inferenceable'] = app['inferenceable']
settings_str = json.dumps(settings).replace("'", "''")
parameters_str = json.dumps(app['parameters']).replace("'", "''")
res = "INSERT INTO time_series_group_app(time_series_group_app_uuid, time_series_group_app_name, display_name, settings, parameters, endpoint_meta, endpoint_train, endpoint_inference, stage, type) VALUES (" + \
f"'{app_uuid}', '{app['appName']}', '{app['displayName']}', '{settings_str}', '{parameters_str}', '{app['endpointMeta']}', '{app['endpointTrain']}', '{app['endpointInference']}', 'Public', 'Internal'" + \
");"
self.logger.info('----- INSERT APP STATEMENT -----')
self.logger.info(res)
self.__export_app(app_uuid, res, TARGET_FINAL)
except Exception as e:
self.logger.exception(f'Failed to get app {app_uuid}.', exc_info=e)
def sql_insert_all(self, only_public=True):
try:
if not self.is_super_user():
raise PermissionError(
'You do not have the permission to generate script for all apps.'
)
apps = self.get_all()
if only_public:
apps = filter(lambda app: app['stage'] == 'Public',
apps['value'])
res = "TRUNCATE TABLE time_series_group_app;" + os.linesep
res += "INSERT INTO time_series_group_app(time_series_group_app_uuid, time_series_group_app_name, display_name, settings, parameters, endpoint_meta, endpoint_train, endpoint_inference, stage, type) VALUES "
for app in apps:
settings = {}
if 'description' in app:
settings['description'] = app['description']
if 'show_result' in app:
settings['show_result'] = app['show_result']
if 'alertable' in app:
settings['alertable'] = app['alertable']
if 'trainable' in app:
settings['trainable'] = app['trainable']
if 'inferenceable' in app:
settings['inferenceable'] = app['inferenceable']
settings_str = json.dumps(settings).replace("'", "''")
parameters_str = json.dumps(app['parameters']).replace(
"'", "''")
stmt = os.linesep + f"('{app['appId']}', '{app['appName']}', '{app['displayName']}', '{settings_str}', '{parameters_str}', '{app['endpointMeta']}', '{app['endpointTrain']}', '{app['endpointInference']}', 'Public', 'Internal'),"
res += stmt
res = res.rstrip(',') + ';'
self.logger.info('----- INSERT APPS STATEMENT -----')
self.logger.info(res)
self.__export_all_apps(res)
except Exception as e:
self.logger.exception('Failed to generate script for all apps.',
exc_info=e)
def train_v1(opts):
## log
## devices
devices = DeviceInfo.get_devices(
[int(i) for i in opts.processors.split(',')])
logger.info('devices: {}'.format(devices))
## metric
rmse = RMSE()
loss_metric = [BaseMetric(name='loss')]
loss_metric.append(BaseMetric('nose'))
loss_metric.append(BaseMetric('delta'))
loss_metric.append(BaseMetric('merge'))
## network
# net = resnet18_v2(classes=136)
net = get_mobilenet_v2(multiplier=1.0, classes=opts.num_classes)
if opts.pretrained_name is not None and opts.pretrained_dir is not None:
logger.info('loading pre-trained {} ...'.format(
os.path.join(opts.pretrained_dir, opts.pretrained_name)))
net.load_params(os.path.join(opts.pretrained_dir,
opts.pretrained_name),
ctx=devices,
allow_missing=True)
logger.info('load pre-trained model complete.')
else:
# net.hybridize()
net.initialize(init=mx.init.Normal(sigma=0.01),
force_reinit=True,
ctx=devices)
logger.info('initial params with random data.')
net.collect_params().reset_ctx(devices)
logger.info('net: {}'.format(net))
## loss
loss_funcs = []
# loss_funcs.append(mx.gluon.loss.SoftmaxCrossEntropyLoss())
loss_funcs.append(mx.gluon.loss.L1Loss())
# loss_funcs.append(L1Loss())
# loss_funcs.append(mx.gluon.loss.L2Loss())
# loss_funcs.append(L2Loss())
## optimizer
lr_scheduler = mx.lr_scheduler.FactorScheduler(step=3,
factor=0.1,
base_lr=opts.learning_rate,
stop_factor_lr=1e-6)
## trainer
trainer = mx.gluon.Trainer(
net.collect_params(),
opts.optimizer,
{
'learning_rate': opts.learning_rate,
# 'momentum': opts,
'wd': opts.weight_decay
# 'lr_scheduler': lr_scheduler
})
## datsets
logger.info('loading datasets ...')
train_iter = RecDataIterV1(rec_dir=opts.rec_dir,
rec_prefix=opts.rec_prefix,
batch_size=opts.batch_size,
shuffle=opts.shuffle)
logger.info('load datasets complete.')
params = net.collect_params()
param_names = params.keys()
logger.info('starting training ...',
show_type=logger.LOG_STYLE.DEFAULT,
forground=logger.LOG_FRONT_COLOR.RED,
background=logger.LOG_BACK_COLOR.DEFAULT)
start_time = time.time()
global_step = 0
for epoch in range(opts.min_epoch, opts.max_epoch):
for idx_step, batch_data in enumerate(iter(train_iter)):
datas = mx.gluon.utils.split_and_load(batch_data.data[0],
ctx_list=devices,
batch_axis=0)
targets = mx.gluon.utils.split_and_load(batch_data.label[0],
ctx_list=devices,
batch_axis=0)
losses = []
with mx.autograd.record():
for datai, targeti in zip(datas, targets):
# print('[===] datai: {} {} {}'.format(datai.shape, datai.dtype, type(datai)))
# print('[===] targeti: {} {} {}'.format(targeti.shape, targeti.dtype, type(targeti)))
targeti = targeti / 128.0
datai = (datai - 127.5) / 128.0
predicts = net(datai)
# print('[===] targeti min-max: {} {}'.format(mx.nd.max(targeti[0, :]).asnumpy()[0], mx.nd.min(targeti[0, :]).asnumpy()[0]))
# print('[===] predicts min-max: {} {}'.format(mx.nd.max(predicts[0, :]).asnumpy()[0], mx.nd.min(predicts[0, :]).asnumpy()[0]))
mse_loss = loss_funcs[0](targeti, predicts)
if False:
points = targeti.reshape((-1, 68, 2))
points = points.asnumpy() * 128.0
image = datai[0, :, :, :].asnumpy() * 128.0 + 127.5
image = np.transpose(image, (1, 2, 0))
image = image.astype(np.uint8)
print('image: {} {}'.format(image.shape, image.dtype))
import cv2
points = points[0, :, :]
for j in range(68):
point = points[j, :]
image = cv2.circle(img=image,
center=(int(round(point[0])),
int(round(point[1]))),
radius=1,
color=(255, 0, 0),
thickness=-1)
import matplotlib.pyplot as plt
plt.imshow(image.get()) ## cv2.UMat --> np.array
# plt.imshow(image) ## cv2.UMat --> np.array
plt.show()
sys.exit(0)
losses.extend(mse_loss)
## update metrics
loss_metric[0].update(mse_loss.shape[0],
mse_loss.asnumpy().sum())
mx.autograd.backward(losses)
# for loss in losses:
# loss.backward()
# mx.autograd.backward(loss)
trainer.step(batch_data.data[0].shape[0])
mx.nd.waitall()
## log
elapse_time = time.time() - start_time
samples_per_second = 0
if global_step > 30:
start_time = time.time()
samples_per_second = int(
opts.batch_size * opts.checkpoint_interval / elapse_time)
if (global_step + 1) % opts.log_interval == 0:
logger.info(
'[{}/{}][{}/{}] [loss: {:.6f} / {:.6f}] [{} samples/s] [lr: {:.10f}]'
.format(epoch, opts.max_epoch - opts.min_epoch, idx_step,
int(train_iter.max_index / opts.batch_size),
loss_metric[0].get_avg()[0],
loss_metric[0].get_avg()[1], samples_per_second,
lr_scheduler.base_lr))
## update metrics
## update global step
global_step += 1
## update trainer
train_iter.reset()
## update learning rate
lr_scheduler(epoch - opts.min_epoch)
trainer.set_learning_rate(lr_scheduler.base_lr)
print('lr: ', lr_scheduler.base_lr, epoch - opts.min_epoch)
## save checkpoint
if (epoch + 1) % opts.checkpoint_interval == 0:
checkpoint_path = os.path.join(
opts.checkpoint_dir,
opts.checkpoint_prefix + '{}'.format(epoch))
logger.info(
'begin save checkpoints {} ...'.format(checkpoint_path),
show_type=logger.LOG_STYLE.DEFAULT,
forground=logger.LOG_FRONT_COLOR.RED,
background=logger.LOG_BACK_COLOR.DEFAULT)
net.save_params(filename=checkpoint_path)
logger.info('complete save checkpoints.',
show_type=logger.LOG_STYLE.DEFAULT,
forground=logger.LOG_FRONT_COLOR.RED,
background=logger.LOG_BACK_COLOR.DEFAULT)
def info(msg: str):
"""
log Info
:param msg:
"""
Logger.info(msg)
def train_v1(opts):
## log
## devices
devices = DeviceInfo.get_devices(
[int(i) for i in opts.processors.split(',')])
logger.info('devices: {}'.format(devices))
## metric
rmse = RMSE()
loss_metric = [BaseMetric(name='loss')]
loss_metric.append(BaseMetric('l1'))
loss_metric.append(BaseMetric('l2'))
loss_metric.append(BaseMetric('l3'))
loss_metric.append(BaseMetric('l4'))
loss_metric.append(BaseMetric('l5'))
loss_metric.append(BaseMetric('l6'))
loss_metric.append(BaseMetric('l7'))
## network
# net = resnet18_v2(classes=136)
net = get_mobilenet_v2(multiplier=1.0, classes=136)
# net.hybridize()
net.initialize(init=mx.init.Normal(sigma=0.01), force_reinit=True)
if opts.pretrained_name is not None and opts.pretrained_dir is not None:
logger.info('loading pre-trained {} ...'.format(
os.path.join(opts.pretrained_dir, opts.pretrained_name)))
net.load_params(os.path.join(opts.pretrained_dir,
opts.pretrained_name),
ctx=devices,
allow_missing=True)
logger.info('load pre-trained model complete.')
net.collect_params().reset_ctx(devices)
logger.info('net: {}'.format(net))
## loss
loss_funcs = []
# loss_funcs.append(mx.gluon.loss.SoftmaxCrossEntropyLoss())
loss_funcs.append(mx.gluon.loss.L1Loss())
# loss_funcs.append(mx.gluon.loss.L2Loss())
# loss_funcs.append(L2Loss())
## optimizer
lr_scheduler = mx.lr_scheduler.FactorScheduler(step=3,
factor=0.1,
base_lr=opts.learning_rate)
## trainer
trainer = mx.gluon.Trainer(
net.collect_params(),
opts.optimizer,
{
'learning_rate': opts.learning_rate,
# 'momentum': opts,
'wd': opts.weight_decay
# 'lr_scheduler': lr_scheduler
})
## datsets
logger.info('loading datasets ...')
train_iter = RecDataIterV1(rec_dir=opts.rec_dir,
rec_prefix=opts.rec_prefix,
batch_size=opts.batch_size,
shuffle=opts.shuffle)
logger.info('load datasets complete.')
params = net.collect_params()
param_names = params.keys()
logger.info('starting training ...',
show_type=logger.LOG_STYLE.DEFAULT,
forground=logger.LOG_FRONT_COLOR.RED,
background=logger.LOG_BACK_COLOR.DEFAULT)
start_time = time.time()
global_step = 0
for epoch in range(opts.min_epoch, opts.max_epoch):
for idx_step, batch_data in enumerate(iter(train_iter)):
datas = mx.gluon.utils.split_and_load(batch_data.data[0],
ctx_list=devices,
batch_axis=0)
targets = mx.gluon.utils.split_and_load(batch_data.label[0],
ctx_list=devices,
batch_axis=0)
losses = []
with mx.autograd.record():
for datai, targeti in zip(datas, targets):
# print('[===] datai: {} {} {}'.format(datai.shape, datai.dtype, type(datai)))
# print('[===] targeti: {} {} {}'.format(targeti.shape, targeti.dtype, type(targeti)))
datai = datai / 255.0
# predicts = net(datai)
predicts = net(datai)
# logger.info('predicts: {} {}, {} {}'.format(
# type(predicts), predicts.shape, mx.nd.max(predicts), mx.nd.min(predicts)
# ))
targeti = targeti / 128.0
# print('[===] targeti min-max: {} {}'.format(mx.nd.max(targeti[0, :]).asnumpy()[0], mx.nd.min(targeti[0, :]).asnumpy()[0]))
# print('[===] predicts min-max: {} {}'.format(mx.nd.max(predicts[0, :]).asnumpy()[0], mx.nd.min(predicts[0, :]).asnumpy()[0]))
mse_loss1 = loss_funcs[0](targeti, predicts[0])
mse_loss2 = loss_funcs[0](targeti[:, 0:34], predicts[1])
mse_loss3 = loss_funcs[0](targeti[:, 34:44], predicts[2])
mse_loss4 = loss_funcs[0](targeti[:, 44:54], predicts[3])
mse_loss5 = loss_funcs[0](targeti[:, 54:72], predicts[4])
mse_loss6 = loss_funcs[0](targeti[:, 72:84], predicts[5])
mse_loss7 = loss_funcs[0](targeti[:, 84:96], predicts[6])
mse_loss8 = loss_funcs[0](targeti[:, 96:136], predicts[7])
mse_loss = mse_loss1 + mse_loss2 + mse_loss3 + mse_loss4 + mse_loss5 + mse_loss6 + mse_loss7 + mse_loss8
losses.extend(mse_loss)
# print('target: {}'.format(targeti))
# print('predicts: {}'.format(predicts))
# print('loss: {}'.format(mse_loss.shape))
# print('[===] {}'.format(mse_loss))
# correct_count, num_samples = acc(preds=predicts, targets=targeti)
## update metrics
loss_metric[0].update(mse_loss.shape[0],
mse_loss.asnumpy().sum())
loss_metric[1].update(mse_loss1.shape[0],
mse_loss1.asnumpy().sum())
loss_metric[2].update(mse_loss2.shape[0],
mse_loss2.asnumpy().sum())
loss_metric[3].update(mse_loss3.shape[0],
mse_loss3.asnumpy().sum())
loss_metric[4].update(mse_loss4.shape[0],
mse_loss4.asnumpy().sum())
loss_metric[5].update(mse_loss5.shape[0],
mse_loss5.asnumpy().sum())
loss_metric[6].update(mse_loss6.shape[0],
mse_loss6.asnumpy().sum())
loss_metric[7].update(mse_loss7.shape[0],
mse_loss7.asnumpy().sum())
mx.autograd.backward(losses)
# for loss in losses:
# loss.backward()
# mx.autograd.backward(loss)
trainer.step(batch_data.data[0].shape[0])
mx.nd.waitall()
## log
elapse_time = time.time() - start_time
samples_per_second = 0
if global_step > 30:
start_time = time.time()
samples_per_second = int(
opts.batch_size * opts.checkpoint_interval / elapse_time)
if (global_step + 1) % opts.log_interval == 0:
logger.info(
'[{}/{}][{}/{}] [loss: {:.6f} / {:.6f} / {:.4f} / {:.4f} / {:.4f} / {:.4f} / {:.4f} / {:.4f} / {:.4f}] [{} samples/s] [lr: {:.10f}]'
.format(epoch, opts.max_epoch - opts.min_epoch, idx_step,
int(train_iter.max_index / opts.batch_size),
loss_metric[0].get_avg()[0],
loss_metric[0].get_avg()[1],
loss_metric[1].get_avg()[0],
loss_metric[2].get_avg()[0],
loss_metric[3].get_avg()[0],
loss_metric[4].get_avg()[0],
loss_metric[5].get_avg()[0],
loss_metric[6].get_avg()[0],
loss_metric[7].get_avg()[0], samples_per_second,
lr_scheduler.base_lr))
## update metrics
## update global step
global_step += 1
## update trainer
train_iter.reset()
## update learning rate
lr_scheduler(epoch - opts.min_epoch)
trainer.set_learning_rate(lr_scheduler.base_lr)
print('lr: ', lr_scheduler.base_lr, epoch - opts.min_epoch)
## save checkpoint
if (epoch + 1) % opts.checkpoint_interval == 0:
checkpoint_path = os.path.join(
opts.checkpoint_dir,
opts.checkpoint_prefix + '{}'.format(epoch))
logger.info(
'begin save checkpoints {} ...'.format(checkpoint_path),
show_type=logger.LOG_STYLE.DEFAULT,
forground=logger.LOG_FRONT_COLOR.RED,
background=logger.LOG_BACK_COLOR.DEFAULT)
net.save_params(filename=checkpoint_path)
logger.info('complete save checkpoints.',
show_type=logger.LOG_STYLE.DEFAULT,
forground=logger.LOG_FRONT_COLOR.RED,
background=logger.LOG_BACK_COLOR.DEFAULT)
try:
dw = DWOperation(meta=meta, logger=logger)
try:
check_sql = 'SELECT COUNT(*) FROM %s.DIM_VENDOR_RETAILER WHERE VENDOR_KEY=%s AND RETAILER_KEY=%s' % (
schema_name, str(options.vendor_key), str(options.retailer_key))
count = dw.query_scalar(check_sql)
if count == 0:
insert_sql = 'INSERT INTO %s.DIM_VENDOR_RETAILER (VENDOR_KEY, RETAILER_KEY) VALUES(%s, %s)' % (
schema_name, str(options.vendor_key), str(
options.retailer_key))
dw.execute(insert_sql)
except pyodbc.ProgrammingError:
logger.info('%s.DIM_VENDOR_RETAILER doesn'
't exist, if you run updateSQL before update' %
schema_name)
logger.info(
'vendor_key=%s, retailer_key=%s has been inserted into %s.DIM_VENDOR_RETAILER'
% (options.vendor_key, options.retailer_key, schema_name))
try:
check_sql = 'SELECT COUNT(*) FROM %s.DIM_INCIDENT_ID_START WHERE VENDOR_KEY=%s AND RETAILER_KEY=%s' % (
schema_name, str(options.vendor_key), str(options.retailer_key))
count = dw.query_scalar(check_sql)
if count == 0:
populate_incident_key_start(dw, options.vendor_key,
options.retailer_key)
except pyodbc.ProgrammingError:
logger.info('%s.DIM_INCIDENT_ID_START doesn'
't exist, if you run updateSQL before update' %
class RemoveDuplicatedRows(SparkOperator):
'''Remove duplicated rows from data with selected columns'''
OP_NAME = 'remove-duplicated-rows'
OP_CATEGORY = 'data-transformation'
def __init__(self):
super(RemoveDuplicatedRows, self).__init__()
self.op_input_num = 1
self.op_output_num = 1
self.op_status = OperatorStatus.INIT
self.op_script_location = 'resources/spark_operators/data_transformation/remove_duplicated_rows.py'
self.op_backend = 'spark'
self.columns = None
def init_operator(self, op_json_param):
self.op_json_param = op_json_param
self.op_input_ops = op_json_param['input-ops']
self.op_input_ops_index = op_json_param['input-ops-index']
self.op_running_mode = self.op_json_param[
'running-mode'] if 'running-mode' in self.op_json_param else 'script'
self.op_local = bool(self.op_json_param['local']
) if 'local' in self.op_json_param else True
self.op_working_directory = self.op_json_param[
'op-working-directory'] if 'op-working-directory' in self.op_json_param else None
self.op_logger = Logger(self.op_working_directory +
'/log/import-csv_' +
str(self.op_json_param['op-index']))
self.columns = op_json_param[
'columns'] if 'columns' in op_json_param else None
def run_function_mode(self):
return self.op_status
def run_script_mode(self):
run_command = 'spark-submit --master '
if self.op_local:
run_command = run_command + 'local[2] '
self.op_result.append(self.op_working_directory + 'output/' +
self.op_json_param['op-index'] + '-output')
run_command = run_command + self.op_script_location + ' ' + self.op_input_ops[
0].get_result(self.op_input_ops_index[0]
) + ' ' + self.op_result[0] + ' ' + self.columns
sub_proc = subprocess.Popen(run_command,
stdout=subprocess.PIPE,
shell=True)
for line in iter(sub_proc.stdout.readline, b''):
self.op_logger.info(line)
sub_proc.stdout.close()
sub_proc.wait()
return sub_proc.returncode
def azkaban_script(self):
run_command = 'spark-submit --master '
if self.op_local:
run_command = run_command + 'local[2] '
self.op_result.append(self.op_working_directory + 'output/' +
self.op_json_param['op-index'] + '-output')
run_command = run_command + self.op_script_location + ' ' + self.op_input_ops[
0].get_result(self.op_input_ops_index[0]
) + ' ' + self.op_result[0] + ' ' + self.columns
return run_command
class DealEngine:
def __init__(self, stock_id, db_conn = None, redis_conn = None):
self.stock_id = stock_id
self.stock_name = 'Not Set'
self.pair_queue = PairQueue(stock_id)
self.logger = Logger('engine')
self.db_conn = db_conn
self.limit = 2
self.close_price = 10
now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()))
self.stock_name = 'sss'
if redis_conn is None:
self.r = redis.Redis()
else:
self.r = redis_conn
cursor = self.db_conn.cursor()
try:
cursor.execute('select stock_name from stock_set where stock_id=%s',
[str(self.stock_id)])
result = cursor.fetchall()[0]
self.stock_name = result[0]
cursor.execute('select gains,decline,status,close_price from stock_state where stock_id=%s',[str(self.stock_id)])
result = cursor.fetchall()
self.limit = result[0][0]
self.gains = self.limit
self.decline = result[0][1]
status = int(result[0][2])
self.close_price = float(result[0][3])
self.last_price = self.close_price
self.exist = True
self.r.hset(stock_id, 'engine_exist', True)
if status == 1:
self.redis_init(True,True,self.gains,self.decline,self.close_price)
self.on = True
print(str(stock_id) + " is running")
else:
self.redis_init(False,True,self.gains,self.decline,self.close_price)
self.on = False
print(str(stock_id) + " is pending")
# self.last_price = float(self.r.hget(self.stock_id,'newest_price').decode('utf-8'))
# self.close_price = self.last_price
# if self.close_price == 0:
# self.close_price = 10
except Exception as e:
self.redis_init(False,False,0,0,0)
self.close_price = 0
self.on = False
self.last_price = 0
print( str(stock_id) +" fails: " + str(e))
self.exist = False
finally:
self.set_open_price()
# cursor.execute('insert into today_stock (stock_id,stock_name,price,date) values (%s,%s,%s,%s)',
# [self.stock_id, self.stock_name, self.close_price, now])
self.db_conn.commit()
cursor.close()
# self.last_price = 4
# self
def redis_init(self,status,engine_exists,gains,decline,close_price):
mapping = {
'stock_id': self.stock_id,
'stock_name': self.stock_name,
'status': status,
'last_price': close_price,
'newest_price': close_price,
'newest': close_price,
'gains': gains,
'decline': decline,
'engine_exist':engine_exists,
'long_count':0,
'short_count':0
}
self.r.hmset(self.stock_id, mapping)
def set_open_price(self):
cursor = self.db_conn.cursor()
cursor.execute("select max(date) from previous_stock where stock_id = %s",[self.stock_id])
last_date = cursor.fetchall()[0][0]
print(last_date)
cursor.execute("select end_price from previous_stock where stock_id=%s and date=%s",[self.stock_id,last_date])
price = float(cursor.fetchall()[0][0])
self.r.hset(self.stock_id,'newest_price',price)
now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()))
cursor.execute('insert into today_stock (stock_id,stock_name,price,date) values (%s,%s,%s,%s)',[self.stock_id,self.stock_name,price,now])
def is_exist(self):
return self.exist
def on_trading(self):
if str(self.stock_id) == '101':
return True
status = self.r.hget(self.stock_id,'status'.encode('utf-8')).decode('utf-8')
status = bool(status)
if self.on is True and status is False:
print(str(self.stock_id) + ' is stopped')
elif self.on is False and status is True:
print(str(self.stock_id) + ' is started')
self.on = status
if not self.exist:
return False
return self.on
# if self.on:
# return True
# else:
#
# # time.sleep(1)
# cursor = self.db_conn.cursor()
# cursor.execute('select status from stock_state where stock_id=%s',[self.stock_id])
# status = int(cursor.fetchall()[0][0])
# if status ==1 :
# self.on = status
# return True
# else:
# return False
def on_stop(self):
self.on = False
self.r.hset(self.id,'status',False)
pass
def on_resume(self):
self.on = True
self.r.hset(self.id, 'status', True)
def on_close(self):
self.on = False
self.exist = False
self.r.hest(self.id,'engine_exist',False)
self.r.hset(self.id, 'status', False)
def save_deal(self, result,buy_order,sell_order):
cursor = self.db_conn.cursor()
# result.price = 2
# result.buy_id = 1
# result.sell_id = 1
print('-----------------------')
buy_fund_id = 'F' + str(result.buy_id)
sell_fund_id = 'F' + str(result.sell_id)
cursor.execute('select security_account from fund_account_user where username=%s',(sell_fund_id,))
sell_id = cursor.fetchall()[0][0]
cursor.execute('select security_account from fund_account_user where username=%s', (buy_fund_id,))
buy_id = cursor.fetchall()[0][0]
cursor.execute('select enabled_money from fund_account_user where username=%s', [buy_fund_id])
buy_money = float(cursor.fetchall()[0][0])
cursor.execute('select enabled_money from fund_account_user where username=%s', [sell_fund_id])
sell_money = float(cursor.fetchall()[0][0])
buy_money -= result.volume * result.price
sell_money += result.volume * result.price
if buy_money < 0:
return
cursor.execute('select freezing_amount from security_in_account where username=%s', [sell_id])
sell_freezing_security = int(cursor.fetchall()[0][0])
sell_freezing_security -= result.volume
#---------------------------------------------------------------------------------------------------------------
# change security
cursor.execute('select amount from security_in_account where username=%s', [sell_id])
sell_security = int(cursor.fetchall()[0][0])
sell_security -= result.volume
sql = "update security_in_account set amount = %s, freezing_amount = %s where username = '******'" % (sell_security,sell_freezing_security,str(sell_id))
cursor.execute(sql)
# quantity = result.price * result.volume
cursor.execute('select amount from security_in_account where username=%s', [buy_id])
result_c = cursor.fetchall()
if result_c:
buy_security = int(result_c[0][0])
else:
buy_security = 0
buy_security += result.volume
if result_c:
cursor.execute('update security_in_account set amount = %s where username = %s',
[buy_security, buy_id])
else:
cursor.execute('insert into security_in_account (username, security_number,security_name ,amount,total_price,freezing_amount ) values (%s,%s,%s,%s,%s,%s)',
[buy_id,self.stock_id,self.stock_name,buy_security,1000,0])
#---------------------------------------------------------------------------------------------------------------
cursor.execute('select freezing_money from fund_account_user where username=%s', [buy_fund_id])
buy_freezing_money = float(cursor.fetchall()[0][0])
buy_freezing_money -= float(buy_order.get_price()) * result.volume
cursor.execute('select freezing_money from fund_account_user where username=%s', [sell_fund_id])
sell_freezing_money = float(cursor.fetchall()[0][0])
sell_freezing_money -= float(sell_order.get_price()) * result.volume
cursor.execute('update fund_account_user set enabled_money=%s, freezing_money = %s where username=%s',[buy_money,buy_freezing_money,buy_fund_id])
cursor.execute('update fund_account_user set enabled_money=%s, freezing_money = %s where username=%s',[sell_money,sell_freezing_security,sell_fund_id])
# cursor.execute('update fund_account_user set enabled_money=%s where username=%s',[sell_money,result.sell_id])
now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()))
cursor.execute('insert into today_stock (stock_id,stock_name,price,date) values (%s,%s,%s,%s)',[self.stock_id,self.stock_name,result.price,now])
cursor.execute('insert into trade_log (stock_id,buy_id,sell_id,price,volume) values (%s,%s,%s,%s,%s) ',[result.stock_id,buy_fund_id,sell_fund_id,int(result.price),result.volume])
self.db_conn.commit()
cursor.close()
self.r.hset(result.stock_id,'last_price',result.price)
self.r.hset(result.stock_id,'newest_price',result.price)
self.r.hset(result.stock_id,'newest',result.price)
print("deal finished")
def deal(self):
long_order, short_order = self.pair_queue.get_first_order()
if not long_order:
# self.logger.info("No Order Now")
return False
if not short_order:
# self.pair_queue.push(long_order)
# self.logger.info("No Order Now")
return False
a = ctypes.c_double(100000.0)
print('+++')
print(self.close_price)
b = ctypes.c_double(self.close_price)
dll = ctypes.CDLL('./deal/libdeal.so')
dll.Deal.restype = exchange
dll.Deal.argtypes = [
stock,stock, ctypes.c_double, ctypes.c_double
]
# converted_long_order = order_conversion(long_order)
# converted_short_order = order_conversion(short_order)
# # print(converted_long_order.buy_id)
# result = dll.Deal(converted_long_order, converted_short_order, a, b)
# print(result.volume)
# self.save_deal(result, long_order, short_order)
# re_order = regenerate_order(result, long_order, short_order)
# if re_order:
# self.pair_queue.push(re_order)
# else:
# pass
# self.logger.info("Success")
try:
converted_long_order = order_conversion(long_order)
converted_short_order = order_conversion(short_order)
print(long_order.volume)
result = dll.Deal(converted_long_order,converted_short_order,a,b)
if int(result.volume) == 0:
raise ctypes.ArgumentError('deal fail')
print("finish")
print(result.volume)
self.save_deal(result,long_order,short_order)
re_order = regenerate_order(result,long_order,short_order)
if re_order:
print("re_order")
print(re_order)
self.pair_queue.push(re_order)
else:
pass
self.pair_queue.remove(long_order.id,LONG)
self.pair_queue.remove(short_order.id,SHORT)
self.logger.info("Success")
except ctypes.ArgumentError as e:
print("in except: " + str(e))
# self.pair_queue.push(long_order)
# self.pair_queue.push(short_order)
# converted_long_order = order_conversion(long_order)
# converted_short_order = order_conversion(short_order)
#
# # dll = ctypes.CDLL('./deal/libdeal.so')
#
# a = ctypes.c_double(self.limit)
# b = ctypes.c_double(self.last_price)
# # dll.Deal.restype = exchange
# result = dll.Deal(converted_long_order, converted_short_order,a,b)
# self.save_deal(result,long_order,short_order)
# re_order = regenerate_order(result=result)
# # self.pair_queue.push(re_order)
self.logger.info("Success")
return True
def single_run(self):
self.deal()
def run(self):
while True:
if str(self.stock_id) is '101':
print('101' + str(self.on_trading()))
if self.on_trading():
self.deal()
else:
pass
target="console",
vendor_key=options.vendor_key,
retailer_key=options.retailer_key)
dw = DWOperation(meta=meta, logger=logger)
try:
for s in schemas:
dw.execute('''
CREATE SCHEMA IF NOT EXISTS %(schema_name)s DEFAULT INCLUDE SCHEMA PRIVILEGES;
GRANT USAGE ON SCHEMA %(schema_name)s TO read_internalusers;
GRANT SELECT ON SCHEMA %(schema_name)s TO read_internalusers;
GRANT USAGE ON SCHEMA %(schema_name)s TO write_internalusers;
GRANT SELECT,INSERT,UPDATE,DELETE,TRUNCATE ON SCHEMA %(schema_name)s TO write_internalusers;
''' % {'schema_name': s})
logger.info(
'%s is created, and permission has been granted for read_internalusers and write_internalusers.'
% s)
for env in ['uat', 'pp', 'Prod']:
try:
dw.execute('''
GRANT USAGE ON SCHEMA %(schema_name)s TO %(env)sIRISolapvtc;
GRANT SELECT ON SCHEMA %(schema_name)s TO %(env)sIRISolapvtc;
''' % {
'schema_name': s,
'env': env
})
logger.info(
'%(schema_name)s permission has been granted for %(env)sIRISolapvtc.'
% {
'schema_name': s,
init_flag=True,
logger=logger)
run_init.main_process()
except Exception as msg:
logger.warning(msg)
raise
finally:
pass
if __name__ == '__main__':
try:
logger.info(
"Trying to sync feedback data from RDP for vendor:{0} and retailer:{1}"
.format(options.vendor_key, options.retailer_key))
# getting meta
# SEP = os.path.sep
# cwd = os.path.dirname(os.path.realpath(__file__))
# generic_main_file = cwd + SEP + '..' + SEP + '..' + SEP + '..' + SEP + 'script' + SEP + 'main.py'
# CONFIG_FILE = cwd + SEP + '..' + SEP + '..' + SEP + '..' + SEP + 'config' + SEP + 'config.properties'
# exec(open(generic_main_file).read())
# trigger_init_feedback(meta=meta, vendor_key=684, retailer_key=158, debug='Y') # testing
trigger_init_feedback(meta=meta,
vendor_key=options.vendor_key,
retailer_key=options.retailer_key)
finally:
class MasterHandler(RequestHandler, ABC):
executor = ThreadPoolExecutor(MAX_EXECUTOR)
def initialize(self, meta):
self.meta = meta
self.logger = Logger(log_level="info", vendor_key=-1, retailer_key=-1)
self.service_name = None
self.is_sync_service = False
self.is_notifiable = True
def prepare(self):
self.set_header('Content-Type', 'application/json')
self.logger.info("Accepted query arguments: %s" %
self.request.query_arguments)
self.query_arguments = {
k: self.get_query_argument(k)
for k in self.request.query_arguments
}
self.logger.info("Ajusted query arguments: %s" % self.query_arguments)
self.request_body = {}
if 'Content-Type' in self.request.headers and self.request.headers[
"Content-Type"].startswith("application/json"):
self.request_body = json.loads(
self.request.body.decode('utf-8')) # it may get list
self.logger.info("Accepted post: %s" % self.request_body)
osa_backend_base_dir = os.path.join(
os.path.dirname(os.path.realpath(__file__)), "..", "..", "..")
service_config = os.path.join(osa_backend_base_dir, 'common', 'config',
'services.json')
with open(service_config) as f:
services = json.load(f)
self.set_service_properties()
if not self.service_name:
raise ValueError('service_name is empty.')
elif not self.service_name in services:
raise ValueError('%s is not found.' % self.service_name)
self.logger.set_keys(module_name=self.service_name)
self.service = services[self.service_name]
module_name = self.service['module_name']
class_name = self.service['class_name']
module = __import__(module_name, fromlist=[class_name])
self.klass = getattr(module, class_name)
@run_on_executor
@Notifier
def async_main(self, params):
self.cls = self.klass(meta=copy.deepcopy(self.meta),
request_body=copy.deepcopy(params),
logger=self.logger) # init class_name
return eval('self.cls.%s' % self.service['class_main_func']
) # call main func and return result
@Notifier
def sync_main(self, params):
self.cls = self.klass(meta=copy.deepcopy(self.meta),
request_body=copy.deepcopy(params),
logger=self.logger)
return eval('self.cls.%s' % self.service['class_main_func'])
def post(self):
if self.is_sync_service:
msg = self.sync_main(self.request_body)
self.send_response(msg)
else:
self.async_main(self.request_body)
msg = '%s - Running %s...' % (datetime.now(), self.service_name)
self.send_response(msg)
def send_response(self, msg):
job_params = {}
if self.is_notifiable:
job_required_keys = [
'jobId', 'stepId', 'batchId', 'retry', 'groupName'
]
job_params = dict([(k, self.request_body[k])
for k in job_required_keys
if k in self.request_body])
job_params['status'] = StepStatus.RUNNING.value
job_params['message'] = msg
job_params['request_body'] = self.request_body
self.write(json.dumps(job_params))
self.flush()
@abstractmethod
def set_service_properties(self):
'''
[Mandatory] set self.service_name
[Optional] call self.set_as_sync_service() to set self.is_sync_service = True
[Optional] call self.set_as_not_notifiable() to set self.is_notifiable = False
'''
pass
def set_as_sync_service(self):
self.is_sync_service = True
def set_as_not_notifiable(self):
self.is_notifiable = False