def __process(self):
self.__calculateInputDomain()
self.__calculateOutputDomain()
for file_name in os.listdir(self.__tmpDataDir):
symbol = file_name.split('_')[0]
if file_name[len(symbol) + 1:] != self.__fileName:
continue
funcUtil.recordStatus(self.__id,
'sampling %s data ...' % file_name)
print 'sampling %s data ...' % file_name
try:
tmp_path = os.path.join(self.__tmpDataDir, file_name)
with open(tmp_path, 'r') as f:
content = f.read()
content = content.split('\n')
for line in content:
if not line:
continue
self.__com[symbol] += json.loads(line)
os.remove(tmp_path)
except Exception, ex:
print ex
funcUtil.write_log('getDataProcess')
self.__com[symbol].sort(Save.sortByDate)
symbol_data = []
data_list = self.__com[symbol]
data_len = len(data_list)
for index, value in enumerate(data_list[0:-1]):
x = self.__sample(value[1], self.__inputDomain)
funcUtil.recordStatus(
self.__id, 'has sample %d | %d' % (index + 1, data_len))
print 'has sample %d | %d' % (index + 1, data_len)
next_data = data_list[index + 1]
y = self.__sample(next_data, self.__outputDomain)
symbol_data.append((x, y))
x_end = self.__sample(data_list[-1][1], self.__inputDomain)
y_end = [0 for i in range(self.__outputNodes)]
symbol_data.append((x_end, y_end))
funcUtil.recordStatus(self.__id, 'finish sample %s' % file_name)
print 'finish sample %s' % file_name
funcUtil.recordStatus(self.__id, 'start save %s data' % file_name)
print 'start save %s data' % file_name
self.__save(symbol, symbol_data)
funcUtil.recordStatus(self.__id, 'finish saving %s' % file_name)
print 'finish saving %s' % file_name
del self.__com[symbol]
def run(self):
s_time = time.time()
try:
data = self.__process(self.__data)
self.__save(data)
except Exception, ex:
print ex
funcUtil.write_log(self.moduleName)
def __prepare(self):
funcUtil.recordStatus(self.__id, 'preparing dispatch manager ...')
self.__uriQueue, self.__dataQueue, self.__symbolList = self.o_prepare.run(
)
self.__comLock = {}
for symbol in self.__symbolList:
if symbol not in self.__com:
self.__com[symbol] = []
self.__comLock[symbol] = threading.Lock()
try:
tmp_path = os.path.join(self.__tmpDataDir,
symbol + '_' + self.__fileName)
if os.path.exists(tmp_path):
os.remove(tmp_path)
except Exception, ex:
print ex
funcUtil.write_log('deleteTmp')
def run(self):
try:
self.__process()
except Exception, ex:
print ex
funcUtil.write_log('save')
def run(self):
try:
self.__save()
except Exception, ex:
print ex
funcUtil.write_log('saveFail')
def __process(self):
# self.__calculateDomain()
for file_name in os.listdir(self.__tmpDataDir):
symbol = file_name.split('_')[0]
if file_name[len(symbol) + 1:] != self.__fileName:
continue
funcUtil.recordStatus(self.__id,
'sampling %s data ...' % file_name)
print 'sampling %s data ...' % file_name
try:
tmp_path = os.path.join(self.__tmpDataDir, file_name)
with open(tmp_path, 'r') as f:
content = f.read()
content = content.split('\n')
for line in content:
if not line:
continue
self.__com[symbol] += json.loads(line)
os.remove(tmp_path)
except Exception, ex:
print ex
funcUtil.write_log('getDataProcess')
self.__com[symbol].sort(Save.sortByDate)
symbol_data = []
data_list = self.__com[symbol]
data_len = len(data_list)
for index, value in enumerate(data_list[0:-self.__xDays -
self.__yDays + 1]):
tmp_x = [value[1][-1][1]]
for i in range(self.__xDays - 1):
tmp_x.append(data_list[index + i + 1][1][-1][1])
x = tmp_x
# x = self.__sample(value[1])
funcUtil.recordStatus(
self.__id, 'has sample %d | %d' % (index + 1, data_len))
print 'has sample %d | %d' % (index + 1, data_len)
next_data_start = data_list[index + self.__xDays]
next_data_end = data_list[index + self.__yDays + self.__xDays -
1]
y = (next_data_start[1][0][1], next_data_end[1][-1][1])
symbol_data.append((x, y))
tmp_x = [data_list[-self.__xDays][1][-1][1]]
for i in range(self.__xDays - 1):
tmp_x.append(data_list[-self.__xDays + i + 1][1][-1][1])
x_end = tmp_x
y_end = (0, 0)
symbol_data.append((x_end, y_end))
funcUtil.recordStatus(self.__id, 'finish sample %s' % file_name)
print 'finish sample %s' % file_name
funcUtil.recordStatus(self.__id, 'start save %s data' % file_name)
print 'start save %s data' % file_name
self.__save(symbol, symbol_data)
funcUtil.recordStatus(self.__id, 'finish saving %s' % file_name)
print 'finish saving %s' % file_name
del self.__com[symbol]
def run(self):
try:
self.__getUriQueue()
except Exception, ex:
print ex
funcUtil.write_log('Prepare')