def prepareData(self, startDate='2007-01-01'):
# check if target table exist
is_full, last_record_date, start_fetch_date = checkIfIncre(
ConfigQuant, self.sourceStockTechIndiTableName,
self.targetTableName, self.dateField, self.lags, '', False)
# sql statement
tmp_ret_field = [] # technical data field
for field in self.retSeriesField:
tmp_sub_ret_field = list(
map(lambda x: '%s_%dD' % (field, x), self.lags))
tmp_ret_field.extend(tmp_sub_ret_field)
self.retSeriesField = tmp_ret_field
tmp_stock_field = [self.dateField, self.codeField]
tmp_stock_field.extend(self.retSeriesField)
stock_field_str = list(map(lambda x: "`%s`" % x, tmp_stock_field))
stock_field_str = ",".join(stock_field_str)
if is_full == 1: # 全量
self.stockState = "SELECT %s FROM %s where " % (
stock_field_str, self.sourceStockTechIndiTableName)
self.if_exist = 'replace'
elif is_full == 0: # 增量
self.stockState = "SELECT %s FROM %s where `%s` > '%s' and " % (
stock_field_str, self.sourceStockTechIndiTableName,
self.dateField, last_record_date)
self.last_update_date = last_record_date
self.if_exist = 'append'
else: # 不需要跑
self.stockState = ''
self.indexState = ''
def prepareData(self, startDate='2007-01-01'):
# check if target table exist
is_full, last_record_date, start_fetch_date = checkIfIncre(
ConfigQuant, self.sourceTableName, self.targetTableName,
self.dateField, self.lags, self.condition)
# sql statement
tmp_field = [
self.dateField, self.codeField, self.openField, self.highField,
self.lowField, self.closeField, self.volumeField
]
if self.turnoverField != '':
tmp_field.append(self.turnoverField)
tmp_field = list(map(lambda x: "`%s`" % x, tmp_field))
fields = ",".join(tmp_field)
if is_full == 1: # 全量
self.state = "SELECT %s FROM %s where " % (fields,
self.sourceTableName)
if self.condition != '':
self.state = self.state + self.condition + ' and ' # add search condition
self.if_exist = 'replace'
elif is_full == 0: # 增量
self.last_update_date = last_record_date
self.state = "SELECT %s FROM %s where `%s` > '%s' " % (
fields, self.sourceTableName, self.dateField, start_fetch_date)
if self.condition != '':
self.state = self.state + ' and ' + self.condition # add search condition
self.if_exist = 'append'
else: # 不需要跑
self.state = ''
def prepareData(self, startDate='2007-01-01'):
# check if target table exist
is_full, last_record_date, start_fetch_date = checkIfIncre(
ConfigQuant, self.sourceTableName, self.targetTableName,
self.dateField, self.lags, self.condition, self.alignFlag)
# sql statement
fields = ",".join([
self.dateField, self.openField, self.highField, self.lowField,
self.closeField, self.volumeField
])
if is_full == 1: # 全量
self.state = "SELECT %s FROM %s" % (fields, self.sourceTableName)
if self.condition != '':
self.state = self.state + " where " + self.condition # add search condition
self.if_exist = 'replace'
elif is_full == 0: # 增量
self.last_update_date = last_record_date
self.state = "SELECT %s FROM %s where `%s` > '%s'" % (
fields, self.sourceTableName, self.dateField, start_fetch_date)
if self.condition != '':
self.state = self.state + " and " + self.condition # add search condition
self.if_exist = 'append'
else: # 不需要跑
self.state = ''
def prepareData(self, startDate='2007-01-01'):
# check if target table exist
is_full, last_record_date, start_fetch_date = checkIfIncre(
ConfigQuant, self.sourceBasicTableName, self.targetTableName,
self.dateField, self.lags, '', self.alignFlag)
if is_full == 2: # 删除了一天重跑
is_full, last_record_date, start_fetch_date = checkIfIncre(
ConfigQuant, self.sourceBasicTableName, self.targetTableName,
self.dateField, self.lags, '', self.alignFlag)
# sql statement
tmp_basic_field = [self.dateField, self.codeField] # basic data field
tmp_basic_field.extend(self.basicSeriesField)
tmp_basic_field = list(map(lambda x: "`%s`" % x, tmp_basic_field))
tmp_tech_field = [] # technical data field
for field in self.techSeriesFieldWithLag:
tmp_sub_tech_field = list(
map(lambda x: '%s_%dD' % (field, x), self.lags))
tmp_tech_field.extend(tmp_sub_tech_field)
self.techSeriesField = tmp_tech_field
self.techSeriesField.extend(self.techSeriesFieldWithoutLag)
tmp_tech_field = [self.dateField, self.codeField]
tmp_tech_field.extend(self.techSeriesField)
tmp_tech_field = list(map(lambda x: "`%s`" % x, tmp_tech_field))
basic_field_str = ",".join(tmp_basic_field)
tech_field_str = ",".join(tmp_tech_field)
if is_full == 1: # 全量
self.basicState = "SELECT %s FROM %s where " % (
basic_field_str, self.sourceBasicTableName)
self.techState = "SELECT %s FROM %s where " % (
tech_field_str, self.sourceTechnicalTableName)
self.if_exist = 'replace'
elif is_full == 0: # 增量
self.basicState = "SELECT %s FROM %s where `%s` > '%s' and " % (
basic_field_str, self.sourceBasicTableName, self.dateField,
last_record_date)
self.techState = "SELECT %s FROM %s where `%s` > '%s' and " % (
tech_field_str, self.sourceTechnicalTableName, self.dateField,
last_record_date)
self.last_update_date = last_record_date
self.if_exist = 'append'
else: # 不需要跑
self.basicState = ''
self.techState = ''
def prepareData(self, startDate='2007-01-01'):
# check if target table exist
is_full, last_record_date, start_fetch_date = checkIfIncre(ConfigQuant, self.sourceTableName,
self.targetTableName, self.dateField, self.lags, '')
# sql statement
fields = self.dateField + ',' + "`%s`" % self.valueField
if is_full == 1: # 全量
self.state = "SELECT %s FROM %s" % (fields, self.sourceTableName)
self.if_exist = 'replace'
elif is_full == 0: # 增量
self.last_update_date = last_record_date
self.state = "SELECT %s FROM %s where `%s` > '%s'" % (
fields, self.sourceTableName, self.dateField, start_fetch_date)
self.if_exist = 'append'
else: # 不需要跑
self.state = ''
def prepareData(self, startDate='2007-01-01'):
# check if target table exist
is_full, last_record_date, start_fetch_date = checkIfIncre(
ConfigQuant, self.sourceTableName, self.targetTableName,
self.timeStampField, [252], self.condition)
# sql statement
# tmp_field = list(map(lambda x: "`%s`" % x, tmp_field))
# fields = ",".join(tmp_field)
season_end_date = {1: '03-31', 2: '06-30', 3: '09-30', 4: '12-31'}
today_dt = datetime.today()
today_str = datetime.strftime(today_dt, '%Y-%m-%d')
current_year = today_dt.year
if today_str < '%d-%s' % (current_year, season_end_date[1]):
current_season = 4
else:
for tmp_season in range(1, 4):
if (today_str >= '%d-%s' %
(current_year, season_end_date[tmp_season])) and (
today_str < '%d-%s' %
(current_year, season_end_date[tmp_season + 1])):
current_season = tmp_season
break
current_period = current_year * 100 + current_season
last_year_period = (
current_year - 2
) * 100 + current_season # load 2 years' old data in order to calculate ttm and yoy (last year's yoy need the data of 2 years ago), and might have nan filled by new downloads
if is_full == 1: # 全量
self.state = "SELECT * FROM %s where " % self.sourceTableName
if self.condition != '':
self.state = self.state + self.condition + ' and ' # add search condition
self.if_exist = 'replace'
# elif is_full == 0: # 增量
else: # even if source do not have new data, still update target
self.last_update_date = datetime.strftime(last_record_date,
'%Y-%m-%d')
self.state = "SELECT * FROM %s where (`%s` * 100 + `%s` >= '%d') and (`%s` * 100 + `%s` <= '%d') " % (
self.sourceTableName, self.yearField, self.seasonField,
last_year_period, self.yearField, self.seasonField,
current_period)
if self.condition != '':
self.state = self.state + ' and ' + self.condition # add search condition
self.if_exist = 'append'