def build_latest_image(container_name, container_image, cont_root=None):
df_name = "Dockerfile-" + dev_type()
if cont_root is None:
cont_root = odb.cache.all_containers.get(container_name, "")
if fexists(pjoin(cont_root, "Dockerfile")):
to_host(pjoin(cont_root, "Dockerfile"))
run(" docker build -t %s . " % (container_image))
elif fexists(pjoin(cont_root, df_name)):
to_host(pjoin(cont_root, df_name), rename="Dockerfile")
run(" docker build -t %s . " % (container_image))
def create(path):
'''
create directory
'''
if not fexists(path):
os.makedirs(path)
os.chmod(path, stat.S_IWRITE | stat.S_IREAD)
def create(path):
'''
create directory
'''
if not fexists(path):
os.makedirs(path)
os.chmod(path, stat.S_IWRITE | stat.S_IREAD)
def mkfile(filename, override=False, mode='w'):
"""Create directories and open file if not existing or override is True.
"""
# Create directory if it does not exit
directory = fdirname(filename)
if directory:
if not fexists(directory):
mkdir(directory)
# Check for existing file if overide is False
if not override:
if fexists(filename):
raise OSError('file exists.')
# Return file object
return open(filename, mode)
def mkfile(filename, override=False, mode='w'):
"""Create directories and open file if not existing or override is True.
"""
# Create directory if it does not exit
directory = fdirname(filename)
if directory:
if not fexists(directory):
mkdir(directory)
# Check for existing file if overide is False
if not override:
if fexists(filename):
raise OSError('file exists.')
# Return file object
return open(filename, mode)
def __init__(self, files=None):
files = files or []
self.files = []
for fname in files:
if not fname:
continue
if fexists(fname):
self.read(fname)
else:
logger.warn('Unable to read config file %s' % fname)
def gen_conts_list():
if "all_containers" not in odb.cache:
odb.cache.all_containers = EasyDict()
for croot_ in odb.pth.docker_containers_root:
if fexists(croot_):
for pth_ in os.listdir(croot_):
cont_path = pjoin(croot_, pth_)
if not os.path.isdir(cont_path):
continue
if not os.path.isfile(pjoin(cont_path, "container_main.py")):
continue
odb.cache.all_containers[pth_] = cont_path
def remove_office_duplicates(file_path):
current_folder = getcwd()
if not ("/" in file_path or "\\" in file_path):
file_path = fjoin(current_folder, file_path)
if not fexists(file_path) or not isfile(file_path):
print "File does not exist: %s" % file_path
return False
print "\nFixing: %s" % file_path
with open(file_path, 'r+') as the_f:
data = the_f.read()
soup = BeautifulSoup(data, "lxml")
divs_list = soup.find_all('div')
# import ipdb
# ipdb.set_trace()
duplicate_total_num = 0
prev_div = None
for div in divs_list:
# print len(div.contents)
try:
if prev_div['style'] == div['style']:
prev_div.replace_with(div)
duplicate_total_num+=1
# print div['style']
except (KeyError, TypeError):
pass
prev_div=div
if duplicate_total_num:
print "%s duplicate borders are removed" % duplicate_total_num
the_f.seek(0)
the_f.write(str(soup))
the_f.truncate()
else:
print "No duplicate borders were found."
def __init__(self, pfFile, verbose=True):
from os.path import exists as fexists
if not fexists(pfFile):
raise RuntimeError('File {0:s} could not be found'.format(pfFile))
self.pfFile = pfFile
self.format = None
self.params = None
self.rotation = False
self.vars = None
self.time = None
self.verbose = verbose
if self.verbose:
print('PFConversion class instance created for file:\n {0:s}'.
format(pfFile))
def handle():
from aer import arguments as args
# TODO try to obtain file_db path from ENV
if not fexists(odb.pth.user_db):
generate_user_db()
# TODO detach file_db and arguments completly
args.init_command_line()
update_fabric_env()
for dbp_ in odb.pth.file_dbs:
load_file_db(dbp_)
args.start_entrypoint()
write_to_db(odb.pth.cache_db, {"cache": odb.cache})
def remove(path):
'''
remove the directory
'''
if fexists(path):
rmtree(path)
def lib_exists(libd):
"""
Returns True/False if the library/project/repository is present on the local dev machine
"""
return fexists(libd.full_path)
def remove(path):
'''
remove the directory
'''
if fexists(path):
rmtree(path)
def rmftree(path):
if not fexists(path):
return
return rmtree(path)
def load_file_db(db_path):
if fexists(db_path):
# print("----",db_path)
with open(db_path) as datafile_db:
data = json.load(datafile_db, object_hook=EasyDict)
transfer_missing_elements(odb, data)
def rmf(*paths):
for path in paths:
if fexists(path):
unlink(path)
def process_run_dir(run_dir, figs=False):
print run_dir
run_data = dict()
# Config file
cfg_file = pjoin(run_dir, 'cfg.json')
if not fexists(cfg_file):
print 'No config file in %s' % run_dir
return
# Get epoch
epoch_file = pjoin(run_dir, 'epoch')
if os.path.exists(epoch_file):
epoch = int(open(epoch_file, 'r').read().strip())
else:
epoch = -1
run_data['epoch'] = epoch
last_cost_file = pjoin(run_dir, 'last_cost')
if os.path.exists(last_cost_file):
run_data['cost'] = float(open(last_cost_file, 'r').read())
# Alive / not
log_file = pjoin(run_dir, 'train.log')
run_data['alive'] = file_alive(log_file, max_dur_sec=60*60)
# Complete / not
run_data['complete'] = os.path.exists(pjoin(run_dir, 'sentinel'))
if run_data['complete']:
run_data['alive'] = "<span style='background:#ccc;'>False</span>"
elif run_data['alive']:
run_data['alive'] = "<span style='background:#6d6;color:#fff'>True</span>"
else:
run_data['alive'] = "<span style='background:#d66;color:#fff'>False</span>"
run_data['run'] = os.path.basename(run_dir)
num_files_file = pjoin(run_dir, 'num_files')
if os.path.exists(num_files_file):
run_data['num_files'] = open(num_files_file, 'r').read()
read_cfg(cfg_file, run_data)
# TODO Load CER and WER
if figs and os.path.exists(pjoin(run_dir, 'params.pk')):
plot_file = pjoin(run_dir, 'plot.png')
cmd = 'python plot_results.py %s --out_file %s' % (run_dir, plot_file)
# Check if params file has been modified after the plot image file
params_file = pjoin(run_dir, 'params.pk')
if (not os.path.exists(plot_file)) or (last_modified(plot_file) < last_modified(params_file)):
print '%s modified, generating plot' % params_file
try:
check_call(cmd, shell=True)
except:
pass
if args.viewer_dir:
plot_dir = pjoin(args.viewer_dir, 'plots')
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
if os.path.exists(pjoin(run_dir, 'plot.png')):
shutil.copyfile(pjoin(run_dir, 'plot.png'),
pjoin(plot_dir, '%s.png' % run_data['run']))
return run_data
def server(host, server_port):
"""
Implements server side TCP socket. The sever accepts only HTTP GET
method requests and responds to requests from the Chrominium browser
(and most other major browsers). Recieve the host name and port number
from the main function.
"""
accepted_ftypes = ["html", "htm", "jpeg", "jpg", "txt"]
server_socket = socket(AF_INET, SOCK_STREAM)
server_socket.bind((host, server_port))
server_socket.listen(1)
print("The server is ready to recieve.\n")
while True:
connection_socket, addr = server_socket.accept()
http_request = connection_socket.recv(8192)
# Regex splits first two lines into three groups
# (GET) (/pathname) (HTTP/1.1)
try:
m = re.match(r"(.*) (.*) (.*)", http_request.decode())
method = m.group(1).strip().upper() # GET
fpath = m.group(2).strip() # /pathname
# Content type
ftype = re.findall(r"([.].*)", fpath)[0][1:].lower()
version = m.group(3).strip().upper() # HTTP/1.1
d = re.match(r"If-Modified-Since:(.*)", http_request.decode())
if d: cdate = d.group(1).strip() # Last modified
# Malformed request message
except:
# Make response message
response_msg = "400 Bad Request\r\n"
response_msg += "Host: Simple-Server\r\n"
datetime = time.strftime("%a, %d %b %Y %H:%M:%S", time.localtime())
response_msg += "Date: " + str(datetime) + "\r\n"
response_msg += "Connection: close\r\n\r\n"
# Send message and display to user
connection_socket.send(response_msg.encode())
connection_socket.send(b"400 Bad Request!")
continue
# Ensure correct HTTP method, version of HTTP, and file path existance
# Everything OK
if (method == "GET" and version == "HTTP/1.1" and
fexists(CURRENT_DIRECTORY + fpath) and ftype in accepted_ftypes):
# Read the file in binary mode
f = open(CURRENT_DIRECTORY + fpath, "rb")
page = f.read()
# Make response message
response_msg = version + " 200 OK\r\n"
response_msg += "Host: Simple-Server\r\n"
datetime = time.strftime("%a, %d %b %Y %H:%M:%S %Z", time.gmtime())
response_msg += "Date: " + str(datetime) + "\r\n"
mdatetime = time.strftime("%a, %d %b %Y %H:%M:%S %Z",
time.gmtime(getmtime(CURRENT_DIRECTORY+fpath)))
response_msg += "Last-Modified: " + str(mdatetime) + "\r\n"
response_msg += "Content-Length: " + str(len(page)) + "\r\n"
cont_type = "text/" if ftype in ["html", "htm", "txt"] else "image/"
ftype = "plain" if ftype == "txt" else ftype
response_msg += "Content-Type: " + cont_type + ftype + "\r\n"
response_msg += "Connection: close\r\n\r\n"
# If-modified-since present
if d:
# Has been modified since; must send
if (time.strptime(cdate, "%a, %d %b %Y %H:%M:%S %Z")
< time.gmtime(getmtime(CURRENT_DIRECTORY+fpath))):
connection_socket.send(response_msg.encode())
connection_socket.send(page)
# Can retrieve from cache
else:
response_msg = version + " 304 Not Modified\r\n"
response_msg += "Date: " + str(datetime) + "\r\n"
response_msg += "Host: Simple-Server\r\n\r\n"
connection_socket.send(response_msg.encode())
# No If-modified-since present; must send
else:
connection_socket.send(response_msg.encode())
connection_socket.send(page)
f.close()
# Incorrect method
elif method != "GET":
# Make response message
response_msg = version + " 501 Not Implemented\r\n"
if method == "BREW":
response_msg = version + " 418 I'm a teapot\r\n"
response_msg += "Host: Simple-Server\r\n"
datetime = time.strftime("%a, %d %b %Y %H:%M:%S", time.localtime())
response_msg += "Date: " + str(datetime) + "\r\n"
response_msg += "Connection: close\r\n\r\n"
# Send message and display to user
connection_socket.send(response_msg.encode())
connection_socket.send(b"418 I'm a teapot!"
if method == "BREW" else b"501 Not Implemented!")
# File does not exist
elif not fexists(CURRENT_DIRECTORY + fpath):
# Make response message
response_msg = version + " 404 Not Found\r\n"
response_msg += "Host: Simple-Server\r\n"
datetime = time.strftime("%a, %d %b %Y %H:%M:%S", time.localtime())
response_msg += "Date: " + str(datetime) + "\r\n"
response_msg += "Connection: close\r\n\r\n"
# Send message and display to user
connection_socket.send(response_msg.encode())
connection_socket.send(b"404 Not Found!")
# Wrong HTTP version
elif not version == "HTTP/1.1":
# Make response message
response_msg = version + " 505 HTTP Version Not Supported\r\n"
response_msg += "Host: Simple-Server\r\n"
datetime = time.strftime("%a, %d %b %Y %H:%M:%S", time.localtime())
response_msg += "Date: " + str(datetime) + "\r\n"
response_msg += "Connection: close\r\n\r\n"
# Send message and display to user
connection_socket.send(response_msg.encode())
connection_socket.send(b"505 HTTP Version Not Supported!")
# Invalid file type
elif ftype not in accepted_ftypes:
# Make response message
response_msg = version + " 415 Unsupported Media Type\r\n"
response_msg += "Host: Simple-Server\r\n"
datetime = time.strftime("%a, %d %b %Y %H:%M:%S", time.localtime())
response_msg += "Date: " + str(datetime) + "\r\n"
response_msg += "Connection: close\r\n\r\n"
# Send message and display to user
connection_socket.send(response_msg.encode())
connection_socket.send(b"415 Unsupported Media Type!")
# Something else wrong
else:
# Make response message
response_msg = "400 Bad Request\r\n"
response_msg += "Host: Simple-Server\r\n"
datetime = time.strftime("%a, %d %b %Y %H:%M:%S", time.localtime())
response_msg += "Date: " + str(datetime) + "\r\n"
response_msg += "Connection: close\r\n\r\n"
# Send message and display to user
connection_socket.send(response_msg.encode())
connection_socket.send(b"400 Bad Request!")
connection_socket.close()
print("Closing the socket.")
