def mkdir(client_pwd, args):
if len(args) != 1:
return "Error not enough arguments! Use: mkdir <argument>"
path = os.path.normpath(format_path(client_pwd) + args[0])
real_path = os.path.normpath(fake_root + path)
if not os.path.exists(real_path):
try:
os.makedirs(real_path)
return ""
except:
return "Unable to create directory!"
else:
return "Directory " + format_path(args[0]) + " already exists!"
def cp(client_pwd, connected_storages, args):
if len(args) != 4:
return "Wrong arguments. Use cp <SOURCE> <DEST>"
remote_path = args[0]
file_name = args[1]
file_size = args[2]
chunks = args[3]
if remote_path == '.':
remote_address = format_path(client_pwd) + file_name
else:
remote_address = (format_path(remote_path) if remote_path.startswith('/') else "/" + format_path(remote_path)) \
+ file_name
return generate_ip_pairs(remote_address, file_size, chunks,
connected_storages)
def plot_closing_adj(path_to_csv):
'''plots the daily adjusted closing price vs. time'''
data = pd.read_csv(format_path(path_to_csv), index_col='date')
print('plotting data for ' + path_to_csv + '...')
print('data dimensions ' + str(data.shape))
plt.plot(data.index.values, data['adjusted'].values)
plt.show()
def init_files(path):
new_path = format_path(path)
headers_with_token = {'Authorization': 'Bearer ' + cache['access_token']}
url = f'https://graph.microsoft.com/v1.0/me/drive/root{new_path}children'
r = requests.get(url, headers=headers_with_token)
file_data = eval(r.text)
if "error" in file_data:
print(file_data['error']['message'])
else:
values = {}
for value in file_data['value']:
if value['name'].lower().strip() == 'readme.md':
readme = requests.get(value['@microsoft.graph.downloadUrl']).text
values['readme.md'] = render_markdown(readme)
elif value['name'].lower().strip() == 'head.md':
readme = requests.get(value['@microsoft.graph.downloadUrl']).text
values['head.md'] = render_markdown(readme)
else:
if 'folder' in value:
folder = {
'time': format_time(value['lastModifiedDateTime']),
'size': format_size(value['size'])
}
values[value['name']] = folder
init_files(path.rstrip('/') + '/' + value['name'])
elif 'file' in value:
file = {
'time': format_time(value['lastModifiedDateTime']),
'size': format_size(value['size']),
'url': value['@microsoft.graph.downloadUrl']
}
values[value['name']] = file
cache_tmp['files'][path] = values
def evaluate_batch(symbols_file, data_path):
'''prep data for evaluating'''
symbols = []
with open(format_path(symbols_file), 'r') as data:
read_data = data.read()
symbols = str(read_data).split()
for symbol in symbols:
test_data = pd.read_csv(format_path(data_path + '/' + symbol + '.csv'),
index_col='date')
model_dir = format_path('output/models/' + symbol)
evaluate(symbol, model_dir, test_data)
print('batch evaluation finished')
def _make_mvn_command_template(self, maven_config):
"""
Construye la plantilla del comando en función de la config.
Esta plantilla acepta los parámetros folder y goal
"""
profiles = ''
# TODO: arreglar el caso de que solo haya un valor (forcelist)
if maven_config.get('profiles'):
profiles = ' '.join(['-P', ','.join(maven_config['profiles'])])
settings = ''
if maven_config.get('settings_file'):
settings = ' '.join(['-s', format_path(maven_config['settings_file'])])
pom_file = ' '.join(['-f', '$folder' + self.pom_filename])
flags_array = ['-B']
if maven_config.get('quiet'):
flags_array.append('-q')
if maven_config.get('skip_tests'):
flags_array.append('-DskipTests')
if maven_config.get('flags'):
flags_array.extend(maven_config['flags'])
flags = ' '.join(flags_array)
base_template_string = ' '.join(['mvn', flags, profiles, settings, pom_file, '$goal'])
return Template(base_template_string)
def stat(client_pwd, args):
if len(args) != 1:
return "Error not enough arguments! Use: stat <argument>"
path = os.path.normpath(format_path(client_pwd) + args[0])
real_path = os.path.normpath(fake_root + path)
if not os.path.exists(real_path):
return "Error path " + path + "not exists!"
else:
if os.path.isdir(real_path):
try:
listdir = os.listdir(real_path)
return "Directory " + path + " contains " + str(
len(listdir)) + " items"
except Exception as e:
return "Unable to get directory stats!"
if os.path.isfile(real_path):
try:
with open(real_path) as f:
file_data = f.readline().split("|")
f.close()
return "File size: " + file_data[
1] + os.linesep + "Divided on: " + file_data[
2] + " chunks"
except:
return "Unable to get file stats!"
def filterOptimizeFiles(self):
self.filter_files = {}
if os.path.isfile(self.path):
head, tail = os.path.split(self.path)
self.filter_files[os.path.normpath(self.path)] = tail
self.path = head
return
for root, dirs, files in scandir.walk(self.path, topdown=True):
dirs[:] = [d for d in dirs if utils.format_path(root, d) not in self.ignore_dirs]
for file in files:
if file.endswith('.py'):
fullpath = utils.format_path(root, file)
relpath = os.path.relpath(fullpath, self.path)
self.filter_files[fullpath] = relpath
def formatPath(self, root):
self.config.IGNORE_DIRS = [
utils.format_path(root, _path) for _path in self.config.IGNORE_DIRS
]
self.config.IGNORE_FILES = [
utils.format_path(root, _path)
for _path in self.config.IGNORE_FILES
]
self.config.INLINE_CONST = [
utils.format_path(root, _path)
for _path in self.config.INLINE_CONST
]
self.config.CONFIG_INLINE_CONST = [
utils.format_path(root, _path)
for _path in self.config.CONFIG_INLINE_CONST
]
self.config.INLINE_CONST_FILES = [
utils.format_path(root, _path)
for _path in self.config.INLINE_CONST_FILES
]
routines = []
for path, sys_path in self.config.SYS_PATH_ROUTINE:
routines.append(
(utils.format_path(root, path),
[utils.format_path(root, _path) for _path in sys_path]))
self.config.SYS_PATH_ROUTINE = routines
def train_batch(symbols_file, data_path, export_dir):
'''prep data for training'''
# read from symbols file
symbols = []
with open(format_path(symbols_file), 'r') as data:
read_data = data.read()
symbols = str(read_data).split()
for symbol in symbols:
print('training neural network model for ' + symbol)
train_data = pd.read_csv(format_path(data_path + '/train/' + symbol + '.csv'), index_col='date')
test_data = pd.read_csv(format_path(data_path + '/test/' + symbol + '.csv'), index_col='date')
model_dir = format_path(export_dir + '/' + symbol)
remove_dir(model_dir)
train(train_data, test_data, format_path(model_dir))
print('training finished for ' + symbol)
def cp(response, temp_dir):
original_file = response
for i, line in enumerate(original_file.splitlines()):
if i == 0:
filepath = line.split('|')[0]
else:
# send file to storage
# format came from chunks_creator
chunk_name = format_path(temp_dir) + os.path.basename(
filepath) + '/chunk_' + str(i) + '.txt'
chunk_data = format_path(filepath) + 'chunk_' + str(
i) + '.txt' + os.linesep
with open(chunk_name, 'r') as chunk:
chunk_data += chunk.read() + os.linesep
send_file_to_storage(line.split('|')[0], chunk_data)
# send copy of file to storage
if (line.split('|')[1] != ''):
send_file_to_storage(line.split('|')[1], chunk_data)
def ls(client_pwd, args):
if len(args) != 1:
return "Error not enough arguments! Use: ls [<arguments>]"
path = os.path.normpath(format_path(client_pwd) + args[0])
real_path = os.path.normpath(fake_root + path)
try:
listdir = os.listdir(real_path)
except Exception as e:
return "Error incorrect path!!"
return os.linesep.join(str(name) for name in listdir)
def mkdir(path):
path = os.path.normpath(path)
real_path = os.path.normpath(fake_root + path)
if not os.path.exists(real_path):
try:
os.makedirs(real_path)
return ""
except:
return "Unable to create directory!"
else:
return "Directory " + format_path(path) + " already exists!"
def cd(client_pwd, args):
if len(args) != 1:
return "Error not enough arguments! Use: cd <argument>"
path = os.path.normpath(format_path(client_pwd) + args[0])
real_path = os.path.normpath(fake_root + path)
if os.path.exists(real_path):
os.chdir(real_path)
return path
else:
return client_pwd
def copy_config():
config = utils.get_json_file_data('config.json')
planConfigRootPath = utils.format_path(
os.path.join(os.getcwd(), config['planConfigDir']))
serverSrcPath = utils.format_path(
os.path.join(planConfigRootPath, config['serverOnlyDirName']))
commonSrcPath = utils.format_path(
os.path.join(planConfigRootPath, config['commonDirName']))
activitiesSrcPath = utils.format_path(
os.path.join(planConfigRootPath, config['activitiesDirName']))
dstPath = utils.format_path(
os.path.join(os.getcwd(), config['serverConfigDir']))
filterList = []
for filterFile in config['serverNotCopy']:
filterPath = utils.format_path(
os.path.join(planConfigRootPath, filterFile))
filterList.append(filterPath)
utils.logger_info("更新配置......")
os.system("svn update " + (planConfigRootPath).encode('gbk'))
utils.logger_info("配置更新完成，准备拷贝！")
utils.logger("配置源目录：")
utils.logger(serverSrcPath.encode('utf-8'))
utils.logger(commonSrcPath.encode('utf-8'))
utils.logger(activitiesSrcPath.encode('utf-8'))
utils.logger("拷贝到：" + dstPath.encode('utf-8'))
utils.cover_copy_files(commonSrcPath, dstPath, filterList)
utils.cover_copy_files(serverSrcPath, dstPath, filterList)
utils.cover_copy_files(activitiesSrcPath, dstPath, filterList)
utils.logger_info('配置拷贝结束！')
def fetch(symbols_file, indicators_file, output_path):
'''fetches stock data combined with technical indicators, output as csv'''
# read from symbols file
stocks = []
with open(utils.format_path(symbols_file), 'r') as data:
read_data = data.read()
stocks = str(read_data).split()
# read from indicators file
indicators = []
with open(utils.1(indicators_file), 'r') as data:
read_data = data.read()
indicators = str(read_data).split()
def preprocess_batch(input_path, output_path, train_ratio):
'''perform preprocessing on all input data csvs'''
start = time.time()
files = get_filename_list(input_path, 'csv')
for file in files:
symbol = file.split('.')[0]
print("preprocessing " + symbol)
data = pd.read_csv(format_path(input_path + '/' + file), index_col='date')
train_data, test_data = preprocess(data, train_ratio)
formatted_output = format_path(output_path)
make_dir_if_not_exists(formatted_output + '/train')
make_dir_if_not_exists(formatted_output + '/test')
train_data.to_csv(formatted_output + '/train' + '/' + symbol + '.csv')
test_data.to_csv(formatted_output + '/test' + '/' + symbol + '.csv')
print('saved csv files to ' + formatted_output + '{train, test}/' + symbol + '.csv')
print("preprocessing complete")
elapsed = time.time() - start
print('time elapsed: ' + str(round(elapsed, 2)) + " seconds")
def cat(client_pwd, connected_storages, args):
if len(args) != 1:
return "Error not enough arguments! Use: stat <argument>"
# file in the pwd directory
filepath = os.path.normpath(format_path(client_pwd) + args[0])
real_path = os.path.normpath(fake_root + filepath)
if not os.path.exists(real_path):
return "Error path %s not exists!" % filepath
else:
with open(real_path, 'r') as read_file:
full_file = ''
lcount = 0
for i, line in enumerate(read_file):
print '%s %s' % (i, line)
if i == 0:
remote_path = line.split('|')[0]
chunks = line.split('|')[2]
print "chunks = ", chunks
if i > 0:
chunk_path = format_path(remote_path) + 'chunk_' + str(
i) + '.txt'
if ip_in_list(line.split('|')[0], connected_storages):
full_file += request_chunk_from_storage(
line.split('|')[0], chunk_path)
elif ip_in_list(line.split('|')[1], connected_storages):
full_file += request_chunk_from_storage(
line.split('|')[1], chunk_path)
else:
return "Some chunk of %s file is not exist" % filepath
lcount += 1
print lcount
if lcount != int(chunks):
return "Some chunk of %s file is not exist" % filepath
else:
return full_file
def get_build_command(self, project_folder):
base_command = "svn"
result_command = ""
project_folder = format_path(project_folder)
if 'revert' in self.svn_config:
result_command += base_command + " revert -R " + project_folder + "; "
params = []
if 'force_update' in self.svn_config:
params.append('update --force')
elif 'update' in self.svn_config:
params.append('update')
result_command += " ".join([base_command, *params, project_folder])
return result_command
def rm(client_pwd, args):
if len(args) != 1:
return "Wrong number of arguments for rm"
path = os.path.normpath(format_path(client_pwd) + args[0])
real_path = os.path.normpath(fake_root + path)
if not os.path.exists(real_path):
return "Error path " + path + "not exists!"
else:
if os.path.isdir(real_path):
try:
shutil.rmtree(real_path)
return ""
except Exception as e:
return "Unable remove directory"
if os.path.isfile(real_path):
try:
os.remove(real_path)
return ""
except:
return "Unable to remove file"
def create_chunks(file_object, temp_dir, chunk_size=1024):
# Cause a single word can not be split into different chunks, count of bytes
# could be changed according to the last word in previous chunk
work_chunk_size = chunk_size
with open(file_object, "rb") as in_file:
# counter for out files
num = 1
# last word from previous chunk (if it was split)
last_word = b''
while True:
chunk = last_word + in_file.read(work_chunk_size)
if not chunk:
break # end of file
# the next byte shows, is the last word in chunk full or not
ch = in_file.read(1)
if ch in [b' ', b'\n', b'\r', b'\t', b'']:
last_word = ch
work_chunk_size = chunk_size - 1
else:
last_word = bytes(re.split(' |\r\n|\n|\t', unicode(chunk))[-1])
if last_word != chunk:
chunk = chunk[0:len(chunk) - len(last_word)]
last_word += ch # add next byte
work_chunk_size = chunk_size - len(last_word) - 1
else:
last_word = ch
work_chunk_size = chunk_size
# create chunk in the temp directory with the path as original file
chunk_name = format_path(temp_dir) + os.path.basename(
file_object) + '/chunk_' + str(num) + '.txt'
if not os.path.exists(os.path.dirname(chunk_name)):
os.makedirs(os.path.dirname(chunk_name))
with open(chunk_name, "wb") as out_file:
out_file.write(chunk)
num = num + 1
return num - 1
# compress_flag=0
cmd = ''
if compress_flag <= 0:
logger.info("Uncompress file %s" % input_file)
target_dir = '.'
if len(args) > 1:
if os.path.isfile(args[1]):
logger.error(
'"%s" is a file which should be a directory, please check')
sys.exit()
target_dir = args[1]
else:
if compress_flag == -1:
target_dir = fname
target_dir = utils.format_path(target_dir)
if not os.path.exists(target_dir):
os.makedirs(target_dir) #create a directory using the filename
fext = fext.lower()
if fext in tar_name_list:
compr = get_compr_app_cmd(fext)
cmd += 'tar %sxvf "%s" -C "%s"' % (compr, input_file, target_dir)
elif fext in zip_name_list:
cmd += 'unzip "%s" -d "%s"' % (input_file, target_dir)
elif compress_flag > 0:
logger.info("Compress file %s" % input_file)
#if len(extension)==0:
#extension='.zip'#default compression extension
if compress_flag == 2:
def get_build_command(self, project_folder):
return self.mvn_command_template.substitute(folder=format_path(project_folder),
goal=self.command_maven_goal)
dfs.append(indicator_data)
stock_indicators_joined = reduce(
lambda left, right:
pd.merge(
left,
right,
left_index=True,
right_index=True,
how='outer'
), dfs)
stock_indicators_joined.index.name = 'date'
# print(stock_indicators_joined)
print('fetched and joined data for ' + stock)
formatted_output_path = utils.format_path(output_path)
utils.make_dir_if_not_exists(output_path)
stock_indicators_joined.to_csv(
formatted_output_path + '/' + stock + '.csv')
print('saved csv file to ' + formatted_output_path + '/' + stock + '.csv')
elapsed = time.time() - start
print('time elapsed: ' + str(round(elapsed, 2)) + " seconds")
if __name__ == '__main__':
fetch(str(sys.argv[1]), str(sys.argv[2]), str(sys.argv[3]))
#if fext in compression_name_list:
# compress_flag=0
cmd=''
if compress_flag <= 0:
logger.info("Uncompress file %s"%input_file)
target_dir='.'
if len(args)>1:
if os.path.isfile(args[1]):
logger.error('"%s" is a file which should be a directory, please check')
sys.exit()
target_dir=args[1]
else:
if compress_flag==-1:
target_dir=fname
target_dir=utils.format_path(target_dir)
if not os.path.exists(target_dir):
os.makedirs(target_dir)#create a directory using the filename
fext=fext.lower()
if fext in tar_name_list:
compr=get_compr_app_cmd(fext)
cmd+='tar %sxvf "%s" -C "%s"'%(compr,input_file,target_dir)
elif fext in zip_name_list:
cmd+='unzip "%s" -d "%s"'%(input_file,target_dir)
elif compress_flag > 0:
logger.info("Compress file %s"%input_file)
#if len(extension)==0:
#extension='.zip'#default compression extension
if compress_flag ==2 :
def fetch(symbols_file, indicators_file, output_path):
'''fetches stock data combined with technical indicators, output as csv'''
# read from symbols file
stocks = []
with open(utils.format_path(symbols_file), 'r') as data:
read_data = data.read()
stocks = str(read_data).split()
# read from indicators file
indicators = []
with open(utils.format_path(indicators_file), 'r') as data:
read_data = data.read()
indicators = str(read_data).split()
stocks_config = {
'function': constants.TIME_SERIES_DAILY_ADJUSTED,
'output_size': constants.OUTPUTSIZE_FULL,
'data_type': constants.DATATYPE_JSON,
'api_key': constants.API_KEY
}
indicators_config = {
'interval': constants.INTERVAL,
'time_period': constants.TIME_PERIOD,
'series_type': constants.SERIES_TYPE,
'api_key': constants.API_KEY
}
for stock in stocks:
start = time.time()
stock_data = fetch_stock.fetch(stock, stocks_config)
time.sleep(1)
dfs = []
dfs.append(stock_data)
for indicator in indicators:
indicator_data = fetch_indicators.fetch(indicator, stock,
indicators_config)
time.sleep(1)
dfs.append(indicator_data)
stock_indicators_joined = reduce(
lambda left, right: pd.merge(
left, right, left_index=True, right_index=True, how='outer'),
dfs)
stock_indicators_joined.index.name = 'date'
# print(stock_indicators_joined)
print('fetched and joined data for ' + stock)
formatted_output_path = utils.format_path(output_path)
utils.make_dir_if_not_exists(output_path)
stock_indicators_joined.to_csv(formatted_output_path + '/' + stock +
'.csv')
print('saved csv file to ' + formatted_output_path + '/' + stock +
'.csv')
elapsed = time.time() - start
print('time elapsed: ' + str(round(elapsed, 2)) + " seconds")
