class Zip:
"""
Handles zip files
"""
def __init__(self, path):
self.state_manager = StateManager()
self.path = path
self.zip_r = None
self.zip_w = None
self.open_read()
self.open_write()
@staticmethod
def selector_matches(selector):
"""
Checks if the specified selector is in the right format
:param selector: Mode selection string
:return: True if the specified string is in the correct format
"""
regex = r"zip:(.*)$"
x = re.match(regex, selector)
if not x:
return None
return x.group(1)
@handle_failure(log)
def open_read(self):
"""
Open a zip file in read mode
:return:
"""
if self.zip_r:
self.zip_r.close()
self.zip_r = ZipFile(self.path, 'r')
@handle_failure(log)
def open_write(self):
"""
Open a zip file in write mode
:return:
"""
if self.zip_w:
self.zip_w.close()
self.zip_w = ZipFile(self.path, 'a')
@handle_failure(log)
def create_state(self):
"""
Creates a state of the files
:return: Tuple of the current state and the previous state
"""
state = []
self.open_read()
name_list = self.zip_r.namelist()
# add nested folders if they doesn't exist
for file in name_list:
folder_path = str(Path(file).parent) + path.sep
if folder_path not in name_list and not self.zip_r.getinfo(
file).is_dir() and folder_path != '.' + path.sep:
state.append({'path': folder_path[:-1], 'is_directory': True})
# set current state for class_a
for file in name_list:
info = self.zip_r.getinfo(file)
if info.is_dir():
state.append({'path': file[:-1], 'is_directory': True})
else:
state.append({
'path': file,
'is_directory': False,
'size': info.file_size,
'last_modified': self.get_last_modified_time(info)
})
# # add top level folder
# if len(name_list) > 0 and name_list[0].split(path.sep)[0] + path.sep not in name_list:
# state.insert(0, {'path': name_list[0].split(
# path.sep)[0], 'is_directory': True})
self.state_manager.set_state(state)
return self.state_manager.get_current_state(
), self.state_manager.get_previous_state()
@handle_failure(log)
def get_last_modified_time(self, zip_info):
"""
Get the last modified time
:param zip_info: ZipInfo object
:return: Last modified time
"""
# (year, month, day, hour, minute, second) = zip_info.date_time
date_string = ''
for item in zip_info.date_time:
date_string = date_string + str(item)
date_time_obj = datetime.datetime.strptime(date_string, '%Y%m%d%H%M%S')
return math.floor(date_time_obj.timestamp() / 2)
@handle_failure(log)
def read(self, filename):
"""
Read the contents of a file
:param filename: File path
:return: The contents of the specified file
"""
for item in self.zip_r.infolist():
if item.filename == filename:
return self.zip_r.read(item.filename)
@handle_failure(log)
def file_exists(self, filename):
"""
Check the existence of a file
:param filename: File path
:return: True if the file exists
"""
for item in self.zip_r.infolist():
if item.filename == filename:
return True
return False
@handle_failure(log)
def is_directory(self, filename):
"""
Checks to see whether the file specified is a directory
:param filename: File path
:return: True if the specified file is a directory
"""
current_state = self.state_manager.get_current_state()
for item in current_state:
if item['path'] == filename:
return item['is_directory']
@handle_failure(log)
def create_directory(self, filename):
"""
Create a new directory
:param filename: Directory path
"""
zfi = ZipInfo(filename + path.sep)
self.zip_w.writestr(zfi, '')
log('Creating directory', filename)
self.open_read()
self.open_write()
@handle_failure(log)
def delete(self, filename):
"""
Delete a file or directory
:param filename: File pth to be deleted
"""
is_directory = self.is_directory(filename)
log("Delete ", is_directory, filename)
zout = ZipFile('./temp.zip', 'w')
for item in self.zip_r.infolist():
buff = self.zip_r.read(item.filename)
if item.filename != filename and (
is_directory and not item.filename.startswith(filename)):
zout.writestr(item, buff)
zout.close()
rename("./temp.zip", self.path)
self.open_read()
@handle_failure(log)
def write(self, filename, content):
"""
Write contents to a file
:param filename: File
:param content: Contents to be written
"""
# zout = ZipFile('./temp.zip', 'w')
# did_write = False
# for item in self.zip_r.infolist():
# buff = self.zip_r.read(item.filename)
# if item.filename == filename:
# zout.writestr(item, content)
# did_write = True
# else:
# zout.writestr(item, buff)
# if not did_write:
# zout.writestr(filename, content)
# zout.close()
# rename("./temp.zip",
# self.path)
# self.open_read()
with ZipFile(self.path, 'w') as zipped_f:
zipped_f.writestr(filename, content)
@handle_failure(log)
def copy_from(self, class_b, filename):
"""
Copy filename from class_b
:param class_b: Class instance that will provide the contents of the file
:param filename: File path
"""
target_path = filename
from_path = filename
if class_b.is_directory(from_path):
log('Copy from ', from_path, 'to', target_path)
self.create_directory(target_path)
else:
log('Copy from ', from_path, 'to', target_path)
contents = class_b.read(from_path)
self.write(filename, contents)
def create_file_hash(self, filename):
"""
create a hash of a file
"""
with tempfile.TemporaryDirectory() as dirpath:
tempfilename = path.join(dirpath, filename)
with open(tempfilename, 'wb') as f:
f.write(self.read(filename))
return create_hash(tempfilename)
class Filesystem:
"""
Handles filesystem files
"""
def __init__(self, target_path):
self.state_manager = StateManager()
if target_path[-1] != path.sep:
target_path = target_path + path.sep
self.path = target_path
@staticmethod
def selector_matches(selector):
"""
Checks if the specified selector is in the right format
:param selector: Mode selection string
:return: True if the specified string is in the correct format
"""
regex = r"folder:(.*)"
x = re.match(regex, selector)
if not x:
return None
return x.group(1)
@handle_failure(log)
def create_state(self):
"""
Creates a state of the files
:return: Tuple of the current state and the previous state
"""
# set current state for class_a
state = []
file_list = list(walk(self.path))
for (index, (dirname, _, filenames)) in enumerate(file_list):
dirname = dirname.replace(self.path, '')
if index > 0:
state.append({'path': dirname, 'is_directory': True})
for filename in filenames:
path_with_file = path.join(dirname, filename)
state.append({
'path':
path_with_file,
'is_directory':
False,
'size':
self.get_file_size(path_with_file),
'last_modified':
self.get_last_modified_time(
path.join(self.path, path_with_file))
})
self.state_manager.set_state(state)
return self.state_manager.get_current_state(
), self.state_manager.get_previous_state()
@handle_failure(log)
def read(self, filename):
"""
Read the contents of a file
:param filename: File path
:return: The contents of the specified file
"""
filename = path.normpath(self.path + filename)
f = open(filename, "rb")
content = f.read()
f.close()
log('Reading ', filename, 'content', content)
return content
@handle_failure(log)
def is_directory(self, filename):
"""
Checks to see whether the file specified is a directory
:param filename: File path
:return: True if the specified file is a directory
"""
filename = path.normpath(self.path + filename)
return path.isdir(filename)
@handle_failure(log)
def get_last_modified_time(self, filepath):
"""
get the last modified time of a file
"""
return math.floor(path.getmtime(path.abspath(filepath)) / 2)
@handle_failure(log)
def create_directory(self, filename):
"""
Create a new directory
:param filename: Directory path
"""
filename = path.normpath(self.path + filename)
mkdir(filename)
@handle_failure(log)
def file_exists(self, filename):
"""
Check the existence of a file
:param filename: File path
:return: True if the file exists
"""
return path.exists(path.join(self.path, filename))
@handle_failure(log)
def delete(self, filename):
"""
Delete a file or directory
:param filename: File pth to be deleted
"""
file = path.abspath(path.normpath(self.path + filename + path.sep))
log('Delete from ', file, self.is_directory(filename))
if self.is_directory(filename):
rmtree(file)
else:
remove(file)
@handle_failure(log)
def copy_from(self, class_b, filename):
"""
Copy filename from class_b
:param class_b: Class instance that will provide the contents of the file
:param filename: File path
"""
target_path = filename
from_path = filename
log('reading ', from_path, 'from class b')
if class_b.is_directory(from_path):
log('Create dir from ', from_path, 'to', target_path)
self.create_directory(target_path)
else:
log('Copy from ', class_b.path + from_path, 'to',
path.normpath(self.path + target_path))
contents = class_b.read(from_path)
f = open(path.normpath(self.path + target_path), 'wb')
f.write(contents)
f.close()
def create_file_hash(self, filename):
"""
create a hash of a file
"""
return create_hash(path.join(self.path, filename))
def get_file_size(self, filename):
"""
get the size of a file
"""
return path.getsize(path.normpath(self.path + filename))
output_size=board_size * board_size,
hidden_layers=config.HIDDEN_LAYERS,
activation=config.ACTIVATION).float()
anet = ANET_MODEL(anet_nn,
max_cases_in_buffer=config.REPLAY_BUFFER_MAX_SIZE)
optimizer = config.OPTIMIZERS[config.OPTIMIZER](anet_nn.parameters(),
lr=lr)
for episode in range(num_episodes):
while state_manager.game_is_on():
random_number = random.uniform(0, 1)
if random_number < 0.3:
action = state_manager.get_random_action()
else:
search_tree = SearchTree(state_manager.get_current_state(),
exploration_bonus)
action = search_tree.simulate_games_and_find_action(
num_simulations, state_manager, anet)
state_manager.take_action(action)
# --------- LOGGING THE TIME MODELS USE TO RAIN --------------
time_finished = time.time()
time_passed = str(round((time_finished - time_start) / 60, 2))
log_string = l = str(episode) + " of " + str(
num_episodes) + " finished. TIME PASSED (minutes): " + time_passed
utils.logging(folder_results, log_string)
# ------------------------------------------------------------
anet.train_model(optimizer,
class Ftp:
"""
Handles ftp files
"""
def __init__(self, username, password, host, path):
ftp = FTP(host)
try:
ftp.login(user=username, passwd=password)
except:
log('Incorrect credentials')
exit()
log('Connected to {} with username {} and password {} on path {}'.
format(host, username, password, path))
self.ftp = ftp
self.state_manager = StateManager()
self.path = path
@staticmethod
def selector_matches(selector):
"""
Checks if the specified selector is in the right format
:param selector: Mode selection string
:return: True if the specified string is in the correct format
"""
regex = r"ftp:(\S*):([\S]+)@([\S^.]+?)([~.\/].*)"
x = re.search(regex, selector)
if not x:
return None
return {
"user": x.group(1),
"password": x.group(2),
"host": x.group(3),
"path": x.group(4),
}
@handle_failure(log)
def listdir(self, _path):
"""
return files and directory names within a path (directory)
"""
file_list, dirs, nondirs = [], [], []
try:
self.ftp.cwd(_path)
except Exception as exp:
log("the current path is : ", self.ftp.pwd(), exp.__str__(), _path)
return [], []
else:
self.ftp.retrlines('LIST', lambda x: file_list.append(x.split()))
# parse each file info result
for info in file_list:
ls_type, name, size = info[0], ' '.join(info[8:]), info[4]
if ls_type.startswith('d'):
dirs.append(name)
else:
modification_time = self.parse_ftp_date(
self.ftp.voidcmd(f"MDTM " + name).split()[-1])
nondirs.append((name, modification_time, int(size)))
return dirs, nondirs
@handle_failure(log)
def walk(self, filepath=path.sep):
"""
Walk through FTP server's directory tree
"""
dirs, nondirs = self.listdir(filepath)
yield filepath, dirs, nondirs
for name in dirs:
filepath = path.join(filepath, name)
yield from self.walk(filepath)
self.ftp.cwd('..')
filepath = path.dirname(filepath)
def generate_filedata(self):
"""
generat a file state
"""
state = []
files = list(self.walk(self.path))
for (index, (dirname, _, filenames)) in enumerate(files):
dirname = dirname.replace(self.path, '')
# ignore the first directory
if index > 0:
state.append({'path': dirname, 'is_directory': True})
for (filename, last_modified, size) in filenames:
path_with_file = path.join(dirname, filename)
state.append({
'path': path_with_file,
'is_directory': False,
'last_modified': last_modified,
'size': size
})
return state
@handle_failure(log)
def create_state(self):
"""
Creates a state of the files
:return: Tuple of the current state and the previous state
"""
state = self.generate_filedata()
self.state_manager.set_state(state)
return self.state_manager.get_current_state(
), self.state_manager.get_previous_state()
@handle_failure(log)
@retry_function(2)
def parse_ftp_date(self, date_string):
"""
Parse a date string
:param date_string: Date string
:return: Date timestamp
"""
date_time_obj = datetime.datetime.strptime(date_string, '%Y%m%d%H%M%S')
return math.floor(date_time_obj.timestamp() / 2)
@handle_failure(log)
@retry_function(2)
def read(self, filepath):
"""
Read the contents of a file
:param filepath: File path
:return: The contents of the specified file
"""
directory_name, file_name = path.split(
path.normpath(self.path + filepath))
r = BytesIO()
self.ftp.cwd(directory_name)
self.ftp.retrbinary('RETR ' + path.join(directory_name, file_name),
r.write)
self.ftp.cwd(self.path)
contents = r.getvalue()
return contents
@handle_failure(log)
@retry_function(2)
def file_exists(self, filepath):
"""
Check the existence of a file
:param filename: File path
:return: True if the file exists
"""
files = self.generate_filedata()
for file in files:
if filepath == file['path']:
return True
return False
@handle_failure(log)
@retry_function(2)
def create_directory(self, filename):
"""
Create a new directory
:param filename: Directory path
"""
log('Create directory ', filename)
try:
self.ftp.mkd(filename)
except Exception:
# silent fail, directory already exists
return
@handle_failure(log)
@retry_function(2)
def delete(self, filename):
"""
Delete a file or directory
:param filename: File pth to be deleted
"""
if self.is_directory(filename):
return self.ftp.rmd(filename)
return self.ftp.delete(filename)
@handle_failure(log)
def is_directory(self, filename):
"""
Checks to see whether the file specified is a directory
:param filename: File path
:return: True if the specified file is a directory
"""
current_state = self.state_manager.get_current_state()
for item in current_state:
if item['path'] == filename:
return item['is_directory']
@handle_failure(log)
@retry_function(2)
def write(self, filename, content):
"""
Write contents to a file
:param filename: File path
:param content: Content to be written
"""
bio = BytesIO(content)
directory_name, file_name = path.split(path.join(self.path, filename))
self.ftp.cwd(directory_name)
log('Writing', content, ' to', directory_name)
self.ftp.storbinary('STOR ' + file_name, bio)
@handle_failure(log)
@retry_function(2)
def copy_from(self, class_b, filename):
"""
Copy filename from class_b
:param class_b: Class instance that will provide the contents of the file
:param filename: File path
"""
target_path = filename
from_path = filename
if target_path[0] == path.sep:
target_path = '.' + target_path
log('Copy', path.join(class_b.path, from_path), 'to',
path.join(self.path, target_path))
if class_b.is_directory(from_path):
self.create_directory(target_path)
else:
self.write(target_path, class_b.read(from_path))
@handle_failure(log)
@retry_function(2)
def create_file_hash(self, filename):
"""
create a hash of a file
"""
m = hashlib.sha1()
self.ftp.retrbinary('RETR ' + self.path + filename, m.update)
return m.hexdigest()
