def main():
gclient.create_dir_if_not_exist("/galaxy/aa-d/test_from_python")
print(gclient.list_dirs_in_dir("/galaxy/aa-d/"))
print(gclient.list_dirs_in_dir("/home/pslx/Downloads"))
print(gclient.get_attr("/home/pslx/Downloads"))
print(gclient.get_attr("/galaxy/aa-d/test_from_python"))
gclient.create_file_if_not_exist("/galaxy/aa-d/test_from_python/test.txt")
gclient_ext.cp_file("/galaxy/aa-d/test_from_python/test.txt",
"/galaxy/aa-d/test_from_python/test1.txt")
gclient_ext.mv_file("/galaxy/aa-d/test_from_python/test.txt",
"/galaxy/aa-d/test_from_python/test3.txt")
print(gclient.list_dirs_in_dir_recursive("/galaxy/aa-d/test_from_python"))
print(gclient.list_files_in_dir_recursive("/galaxy/aa-d/test_from_python"))
t = time.time()
gclient.read("/galaxy/aa-d/large_test.txt")
print(time.time() - t)
data = gclient.read_multiple(
["/galaxy/aa-d/test_from_python/test1.txt", "/galaxy/aa-d/test3.txt"])
for key, val in data.items():
print(key, val)
data = gclient_ext.read_txts(
["/galaxy/aa-d/test_from_python/test1.txt", "/galaxy/aa-d/test3.txt"])
for key, val in data.items():
print(key, val)
print(gclient.list_cells())
print(gclient.read('/galaxy/aa-d/test.pb'))
print(gclient_ext.read_txt('/galaxy/aa-d/test.pb'))
message = TestMessage()
message.name = "test"
gclient_ext.write_proto_message('/galaxy/aa-d/test1.pb', message)
print(gclient_ext.read_proto_message('/galaxy/aa-d/test1.pb', TestMessage))
print(gclient.check_health("aa"))
gclient.write_multiple(path_data_map={
'/galaxy/aa-d/test_from_python/test2.txt': '123',
'/galaxy/aa-d/test_from_python/test4.txt': '234',
},
mode='a')
gclient_ext.write_proto_messages(
path_data_map={
'/galaxy/aa-d/test_from_python/test2.pb': message,
'/galaxy/aa-d/test_from_python/test4.pb': message,
})
print(
gclient_ext.read_proto_messages(paths=[
'/galaxy/aa-d/test_from_python/test2.pb',
'/galaxy/aa-d/test_from_python/test4.pb'
],
message_type=TestMessage))
print(gclient.create_file_if_not_exist("/LOCAL/test/test.txt"))
print(gclient.create_file_if_not_exist("/galaxy/aa-d/test/test.txt"))
def get_containers_info():
containers_info = []
existing_containers = {}
all_proto_files = set()
if not FileUtil.is_local_path(backend_folder):
all_cells = ['']
else:
all_cells = gclient.list_cells()
for cell_name in all_cells:
folder = FileUtil.convert_local_to_cell_path(
path=backend_folder, cell=cell_name)
proto_files = FileUtil.list_files_in_dir(folder)
all_proto_files = all_proto_files.union(set(proto_files))
for proto_file in all_proto_files:
storage = ProtoTableStorage()
storage.initialize_from_file(
file_name=proto_file
)
raw_data = storage.read_all()
if not raw_data:
continue
key = sorted(raw_data.keys())[-1]
val = raw_data[key]
result_proto = ProtoUtil.any_to_message(
message_type=ContainerBackendValue,
any_message=val
)
ttl = result_proto.ttl
if ttl > 0 and result_proto.updated_time and TimezoneUtil.cur_time_in_pst() - TimezoneUtil.cur_time_from_str(
result_proto.updated_time) >= datetime.timedelta(days=ttl):
FileUtil.remove_file(storage.get_file_name())
else:
container_info = {
'container_name': result_proto.container_name,
'status': ProtoUtil.get_name_by_value(
enum_type=Status, value=result_proto.container_status),
'updated_time': result_proto.updated_time,
'mode': ProtoUtil.get_name_by_value(enum_type=ModeType, value=result_proto.mode),
'data_model': ProtoUtil.get_name_by_value(
enum_type=DataModelType, value=result_proto.data_model),
'run_cell': result_proto.run_cell,
'snapshot_cell': result_proto.snapshot_cell,
}
if container_info['container_name'] not in existing_containers:
existing_containers[container_info['container_name']] = container_info['updated_time']
containers_info.append(container_info)
else:
if container_info['updated_time'] >= existing_containers[container_info['container_name']]:
containers_info.append(container_info)
return containers_info
def _check_health():
for cell in gclient.list_cells():
logger.info("Checking health for cell " + cell + '.')
health_json = gclient.check_health(cell)
try:
health = json.loads(health_json)
if not health['healthy']:
logger.error('Cell ' + cell + ' not healthy!')
else:
logger.info('Cell ' + cell + ' healthy.')
except Exception as err:
logger.error('Checking health for cell ' + cell +
' failed with error ' + str(err) + '.')
def get(self, p=''):
p = p.strip()
logger.info('Getting GET request from ip [' + request.remote_addr +
'].')
if p and p[0] != '/':
p = '/' + p
total = {'size': 0, 'dir': 0, 'file': 0}
contents = []
if p == ROOT or not p:
cells = gclient.list_cells()
for cell in cells:
path_name = os.path.join(ROOT, cell + '-d')
try:
attr = json.loads(gclient.get_attr(path_name))
except Exception as err:
logger.error('Getting attribute of ' + path_name +
' failed with error ' + str(err) + '.')
continue
info = {
'name': path_name,
'mtime': int(attr['mtime']),
'type': 'dir',
'size': 0
}
total['dir'] += 1
contents.append(info)
page = render_template('index.html',
path=ROOT,
contents=contents,
total=total,
hide_dotfile="yes")
res = make_response(page, 200)
else:
if gclient.dir_or_die(p):
contents = []
total = {'size': 0, 'dir': 0, 'file': 0}
for filename, statbuf in gclient_ext.list_all_in_dir(
p).items():
if filename in ignored or filename[0] == '.':
continue
stat_res = json.loads(statbuf)
info = {}
info['name'] = filename
info['mtime'] = int(stat_res['mtime'])
ft = get_type(int(stat_res['mode']))
info['type'] = ft
total[ft] += 1
sz = int(stat_res['size']) if 'size' in stat_res else 0
info['size'] = sz
total['size'] += sz
contents.append(info)
page = render_template('index.html',
path=p,
contents=contents,
total=total,
hide_dotfile="yes")
res = make_response(page, 200)
elif gclient.file_or_die(p):
res = file_response(p)
else:
res = make_response('Not found', 404)
return res
help='The port for the viewer')
parser.add_argument('--cell',
dest='cell',
default='bb',
help='The cell to store the log')
parser.add_argument('--debug',
dest='debug',
action='store_true',
help='Whether to use debug mode')
parser.add_argument('--health_check_interval',
dest='health_check_interval',
default=60,
help='"Interval for checking server health')
args = parser.parse_args()
assert args.username is not None and args.password is not None and args.port is not None, 'Invalid arguments'
assert args.cell in gclient.list_cells(), "Invalid cell"
ROOT = '/galaxy/'
APP_NAME = 'galaxy_viewer'
LOG_DIR = '/LOCAL/ttl=7d/logs/' + APP_NAME + '/'
logger = glogging.get_logger(APP_NAME, LOG_DIR)
galaxy_viewer = Flask(APP_NAME,
static_url_path='/assets',
static_folder='assets')
galaxy_viewer.config.update(SECRET_KEY=APP_NAME)
galaxy_viewer.config.update(SESSION_COOKIE_NAME=APP_NAME + '_cookie')
ignored = [
'.bzr', '$RECYCLE.BIN', '.DAV', '.DS_Store', '.git', '.hg', '.htaccess',