def create_backup(self):
backup_functions = [
self.__enable_maintenance_mode, self.__dump_db,
self.__export_config, self.__disable_maintenance_mode,
self.__tar_backup, self.__set_file_permissions,
self.__delete_tmp_dir
]
if self.__create_backup_dir() and self.__create_tmp_dir():
try:
for fn in backup_functions:
if not fn():
_print(F"{Fore.RED}Backup aborted.{Style.RESET_ALL}")
return False
return round(
Path(self.tar_gz_file_path).stat().st_size / 1000000, 2)
except:
self.exceptions.update({'backup': traceback.format_exc()})
return False
else:
_print(
F"{Fore.RED}Could not create temporary folder or backup folder. Backup aborted.{Style.RESET_ALL}"
)
return False
def sync_song(self):
for song_detail in self.source_playlist:
song = song_detail[0]
singer = song_detail[1]
search_word = u"{} {}".format(song, singer)
url_sw = quote(search_word.encode('utf8'))
self.browser.get(qq_search_url.format(url_sw))
self.wait.until(lambda browser: browser.find_element_by_class_name(
"songlist__list"))
sleep(0.5)
@retry(retry_times=3)
def _add(browser):
browser.execute_script(
"document.getElementsByClassName('songlist__list')[0].firstElementChild.getElementsByClassName('list_menu__add')[0].click()"
)
sleep(0.5)
browser.find_element_by_css_selector(
"a[data-dirid='{}']".format(
self.target_playlist_tag)).click()
_print(u"song:{} success".format(song))
try:
_add(self.browser)
except RetryException:
_print(u"song:{}, sync error".format(song))
self.failed_list.append(search_word)
else:
self.success_list.append(search_word)
def restore_backup(self, backup_file_path) -> bool:
self.backup_file_path = backup_file_path
self.restore_dump_file = os.path.basename(backup_file_path[:-6] +
"sql")
self.restore_dump_file_path = os.path.join(self.tmp_dir,
self.restore_dump_file)
self.restore_tar_file = os.path.basename(backup_file_path[:-3])
self.restore_tar_file_path = os.path.join(self.tmp_dir,
self.restore_tar_file)
restore_functions = [
self.__untar_backup, self.__enable_maintenance_mode,
self.__import_db, self.__import_config,
self.__disable_maintenance_mode, self.__delete_tmp_dir
]
if self.__create_tmp_dir():
try:
for fn in restore_functions:
if not fn():
return False
return True
except:
self.exceptions.update({'restore': traceback.format_exc()})
return False
else:
_print(
F"{Fore.RED}Could not create temporary folder. Restore aborted.{Style.RESET_ALL}"
)
return False
def sync_song(self):
for song_detail in self.source_playlist:
song = song_detail[0]
singer = song_detail[1]
search_word = u"{} {}".format(song, singer)
url_sw = quote(search_word.encode('utf8'))
@retry(retry_times=3)
def _add(self):
self.browser.get(wy_search_url.format(url_sw))
self.browser.switch_to.frame("contentFrame")
WebDriverWait(self.browser, 10).until(
EC.presence_of_element_located(
(By.CSS_SELECTOR, 'span[data-res-action="fav"]')))
self.browser.execute_script(
"document.querySelectorAll('span[data-res-action=\"fav\"]')[0].click()"
)
sleep(0.5)
self.browser.execute_script(
"document.querySelectorAll('li[data-id=\"{}\"]')[0].click()"
.format(self.target_playlist_tag))
sleep(0.5)
_print(u"song:{} success".format(song))
try:
_add(self)
except RetryException:
_print(u"song:{}, sync error".format(song))
self.failed_list.append(search_word)
else:
self.success_list.append(search_word)
def __download(self,lists):
cmds=[]
_print(dir(lists))
_print ("开始下载。。。")
for num in lists:
num=int(num[0])-1
cmd="aria2c -c -s10 -x10 --header 'Cookie:ptisp=edu; FTN5K=%s' '%s'"%(self.filecom[num],self.filehttp[num])
_print (cmd)
if sys.version_info >= (3,0):
pass
else:
cmd=cmd.encode("u8")
cmds.append(cmd)
"""
调用aria2进行下载
"""
_print(cmds)
for i in cmds:
os.system("cd %s && %s"%(self._downpath,i))
try:
subprocess.Popen(["notify-send","xfdown: 下载完成!"])
except:
_print("notify-send error,you should have libnotify-bin installed.")
def finish(self):
_print("Fumador desconectado *{}*".format(store.get(self.code)["name"]))
if self.rejected is False:
store.get(self.code)["flag"] = False
global smoke_code
if smoke_code == self.code:
global smoke
smoke = False
def _add(browser):
browser.execute_script(
"document.getElementsByClassName('songlist__list')[0].firstElementChild.getElementsByClassName('list_menu__add')[0].click()"
)
sleep(0.5)
browser.find_element_by_css_selector(
"a[data-dirid='{}']".format(
self.target_playlist_tag)).click()
_print(u"song:{} success".format(song))
def __delete_tmp_dir(self) -> bool:
try:
shutil.rmtree(self.tmp_dir)
return not os.path.isdir(self.tmp_dir)
except:
_print(
F"{Fore.RED}Could not delete old tmp folder{Style.RESET_ALL}")
self.exceptions.update({'delete_tmp_dir': traceback.format_exc()})
return False
def __untar_backup(self) -> bool:
try:
with tarfile.open(self.backup_file_path, 'r:gz') as tarball:
tarball.extractall(self.tmp_dir)
status = os.path.isdir(os.path.join(self.tmp_dir, "config"))
_print(F"Unzip backup: {self.SUCCESS if status else self.FAILED}")
return status
except:
_print(F"Unzip backup: {self.FAILED}")
self.exceptions.update({'__untar_backup': traceback.format_exc()})
return False
def __set_file_permissions(self) -> bool:
try:
os.chmod(self.tar_gz_file_path, stat.S_IREAD)
status = oct(os.stat(self.tar_gz_file_path).st_mode)[-3:] == '400'
_print(
F"Set secure file permissions: {self.SUCCESS if True else self.FAILED}"
)
return status
except:
_print(F"Set secure file permissions: {self.FAILED}")
self.exceptions.update(
{'__set_file_permissions': traceback.format_exc()})
return False
def __tar_backup(self) -> bool:
try:
with tarfile.open(self.tar_gz_file_path, 'w:gz') as tarball:
tarball.add(self.__dump_file_path, arcname=self.__dump_file)
tarball.add(os.path.join(self.tmp_dir, "config"),
arcname="/config")
status = True # TODO: Implement a test to confirm that files where added to tar file
_print(F"Zip backup: {self.SUCCESS if status else self.FAILED}")
return status
except:
_print(F"Zip backup: {self.FAILED}")
self.exceptions.update({'__tar_backup': traceback.format_exc()})
return False
def __dump_db(self) -> bool:
try:
os.system(
F"docker exec {self.db_container} mysqldump --default-character-set=utf8mb4 --password={self.__password} --all-databases > {self.__dump_file_path}"
)
status = os.path.isfile(self.__dump_file_path)
_print(
F"Dump Nextcloud database: {self.SUCCESS if status else self.FAILED}"
)
return status
except:
_print(F"Dump Nextcloud database: {self.FAILED}")
self.exceptions.update({'__dump_db': traceback.format_exc()})
return False
def init(port):
try:
server = MyTCPServer(("0.0.0.0", port), MyTCPServerHandler)
server.timeout = 10
server_thread = threading.Thread(target=server.serve_forever)
server_thread.timeout = 10
# iniciando agente
_print("Esperando fumadores...")
server_thread.daemon = True
server_thread.start()
while True:
verify_smoking()
global smoke_code
smoke_code = choice(codes)
_print("Agente: Tengo disponible {}!".format(store.get(smoke_code)["required"]))
global smoke
smoke = True
store.get(smoke_code)["request"].send("enable".encode("UTF-8"))
_print("Agente: fumador {} servido!".format(store.get(smoke_code)["name"]))
except KeyboardInterrupt:
_print("Cerrando conexiones...")
server.shutdown()
server.server_close()
def verify_smoking():
# Se verifica si estan todos los fumadores conectados
while True:
active_smokers = True
for i in codes:
if store[i].get("flag") is False:
active_smokers = False
break
time.sleep(time_sleep)
if active_smokers and smoke is False:
break
else:
if active_smokers is False:
_print("Agente: Esperando todos los fumadores")
def log(self, message):
dt = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
entry = dt + " ; " + message + "\n"
if self.__check_log_dir():
try:
with open(self.__log_file_path, "a+") as log_file:
log_file.writelines(entry)
return True
except:
_print(
F"{Fore.RED}Could not write to log file: {self.__log_file}{Style.RESET_ALL}"
)
self.exceptions.update({'log': traceback.format_exc()})
return False
def __check_log_dir(self) -> bool:
if not os.path.isdir(self.log_dir):
try:
os.makedirs(self.log_dir)
return True
except:
_print(
F"{Fore.RED}Could not create log directory under: {self.log_dir}{Style.RESET_ALL}"
)
self.exceptions.update(
{'get_log_file': traceback.format_exc()})
return False
else:
return True
def __create_tmp_dir(self) -> bool:
if not os.path.isdir(self.tmp_dir):
try:
os.mkdir(self.tmp_dir)
return os.path.isdir(self.tmp_dir)
except:
_print(
F"{Fore.RED}Could not create tmp folder{Style.RESET_ALL}")
self.exceptions.update(
{'create_tmp_dir': traceback.format_exc()})
return False
else:
if self.__delete_tmp_dir():
return self.__create_tmp_dir()
def _add(self):
self.browser.get(wy_search_url.format(url_sw))
self.browser.switch_to.frame("contentFrame")
WebDriverWait(self.browser, 10).until(
EC.presence_of_element_located(
(By.CSS_SELECTOR, 'span[data-res-action="fav"]')))
self.browser.execute_script(
"document.querySelectorAll('span[data-res-action=\"fav\"]')[0].click()"
)
sleep(0.5)
self.browser.execute_script(
"document.querySelectorAll('li[data-id=\"{}\"]')[0].click()"
.format(self.target_playlist_tag))
sleep(0.5)
_print(u"song:{} success".format(song))
def __create_backup_dir(self) -> bool:
if not os.path.isdir(self.backup_dir):
try:
os.makedirs(self.backup_dir)
_print(
F"{Fore.GREEN}Backup folder created under: {self.backup_dir}{Style.RESET_ALL}"
)
return True
except:
_print(
F"{Fore.RED}Could not backup tmp folder under {self.backup_dir}{Style.RESET_ALL}"
)
self.exceptions.update(
{'create_backup_dir': traceback.format_exc()})
return False
else:
return True
def process(self):
while True:
message = self.request.recv(packet_size).decode("UTF-8")
if message == "need":
_print("{}: Necesito {}!".format(store.get(self.code)["name"], store.get(self.code)["required"]))
if self.smoke_released:
self.smoke_released = False
global smoke
smoke = False
elif message == "enable":
_print("{}: Termino de fumar!".format(store.get(self.code)["name"]))
self.smoke_released = True
elif message == "ack":
time.sleep(time_smoke)
elif message == "exit":
break
time.sleep(time_sleep)
def __restart_containers(self):
try:
if os.path.isfile(self.docker_compose_file_path):
path = Path(self.docker_compose_file_path)
directory = path.parent
elif os.path.isdir(self.docker_compose_file_path):
directory = self.docker_compose_file_path
else:
raise Exception("Docker Compose path invalid")
os.chdir(directory)
result = check_output(["docker-compose", "up", "-d"])
_print(F"Restart docker containers: {self.SUCCESS}")
return True # TODO: Implement test
except:
self.exceptions.update(
{'__restart_containers': traceback.format_exc()})
_print(F"Restart docker containers: {self.FAILED}")
return False
def init(ip, port, code):
try:
request = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
request.connect((ip, port))
request.send('{}'.format(code).encode('UTF-8'))
time.sleep(time_sleep)
resp = request.recv(packet_size).decode('UTF-8')
if resp == 'accepte':
process(code, request)
else:
_print('Rechazado por el agente.')
request.close()
except KeyboardInterrupt:
_print('Cerrando conexiones...')
request.send('exit'.encode('UTF-8'))
request.close()
def __import_config(self) -> bool:
try:
if os.path.isdir(os.path.join(self.tmp_dir, "config")):
with tarfile.open(self.restore_tar_file_path, 'w') as tarball:
tarball.add(os.path.join(self.tmp_dir, "config"),
arcname="/config")
os.system(
F"docker cp {self.restore_tar_file_path} {self.app_container}:/var/www/html/config.tar"
)
os.system(F"docker exec {self.app_container} rm -r config")
os.system(F"docker exec {self.app_container} tar -xf config.tar")
os.system(F"docker exec {self.app_container} rm config.tar")
status = True # TODO: implement a test if export into docker container was successful
_print(
F"Import Nextcloud configuration: {self.SUCCESS if status else self.FAILED}"
)
return status
except:
_print(F"Import Nextcloud configuration: {self.FAILED}")
self.exceptions.update({'__import_config': traceback.format_exc()})
return False
def __import_db(self) -> bool:
try:
import_db = subprocess.Popen(['cat', self.restore_dump_file_path],
stdout=subprocess.PIPE)
result = check_output([
"docker",
"exec",
self.db_container,
"mysql",
"--user=root",
"--password="******"utf-8")
_print(
F"Import Nextcloud database: {self.SUCCESS if status else self.FAILED}"
)
return status
except subprocess.CalledProcessError as e:
_print(F"Import Nextcloud database: {self.FAILED}")
self.exceptions.update({'__import_db': traceback.format_exc()})
return False
def __pull_images(self):
update_required = False
try:
if os.path.isfile(self.docker_compose_file_path):
path = Path(self.docker_compose_file_path)
directory = path.parent
elif os.path.isdir(self.docker_compose_file_path):
directory = self.docker_compose_file_path
else:
raise Exception("Docker Compose path invalid")
os.chdir(directory)
result = check_output(["docker-compose", "pull"])
result = result.decode("utf-8").split("\n")
for line in result:
if line.startswith("Status:"):
_print(line)
if 'Image is up to date' not in line:
update_required = True
return update_required
except:
self.exceptions.update({'__pull_images': traceback.format_exc()})
_print(F"Pull new docker images: {self.FAILED}")
return False
def cleanup(self):
if not utils.no_cleanup:
deleted_files = 0
backup_dir = os.scandir(self.backup_dir)
backup_files = [
file for file in backup_dir
if file.is_file() and file.name.startswith(self.name)
and file.name.endswith(".tar.gz")
]
while len(backup_files) > self.number_of_backups:
del_file = min(backup_files, key=os.path.getctime)
backup_files.remove(del_file)
os.remove(del_file)
deleted_files += 1
if deleted_files == 1:
_print(
F"{Fore.YELLOW}Deleted 1 old backup file.{Style.RESET_ALL}"
)
elif deleted_files >= 1:
_print(
F"{Fore.YELLOW}Deleted {deleted_files} old backup files.{Style.RESET_ALL}"
)
def process(code, request):
message = ''
while True:
if message != 'ack':
_print('Esperando {}!'.format(store.get(code)['required']))
request.send('need'.encode('UTF-8'))
message = request.recv(packet_size).decode('UTF-8')
if message == 'enable':
_print('Servido!')
time.sleep(time_sleep)
request.send('ack'.encode('UTF-8'))
_print('Armando cigarro!')
time.sleep(time_sleep)
_print('Fumando!!!')
time.sleep(time_smoke)
request.send('enable'.encode('UTF-8'))
elif message == 'ack':
pass
time.sleep(time_sleep)
def handle(self):
# Proceso de reconocimiento
# cur_thread = threading.current_thread()
self.code = self.request.recv(packet_size).decode("UTF-8")
self.rejected = False
self.smoke_released = False
_print("Conectando fumador...")
if store.get(self.code)["flag"] is False:
store.get(self.code)["request"] = self.request
store.get(self.code)["flag"] = True
_print("Fumador aceptado *{}*".format(store.get(self.code)["name"]))
self.request.send("accepte".encode("UTF-8"))
self.process()
else:
self.rejected = True
_print("Fumador rechazado *{}*".format(store.get(self.code)["name"]))
self.request.send("rejected".encode("UTF-8"))
def __enable_maintenance_mode(self) -> bool:
try:
enable_maintenance_mode = check_output([
"docker", "exec", "--user", "www-data", self.app_container,
"php", "occ", "maintenance:mode", "--on"
])
chunks = enable_maintenance_mode.decode("utf-8").split('\n')
if 'Maintenance mode enabled' in chunks:
_print(F"Enable Nextcloud maintenance mode: {self.SUCCESS}")
return True
elif 'Maintenance mode already enabled' in chunks:
return True
else:
_print(F"Enable Nextcloud maintenance mode: {self.FAILED}")
return False
except:
self.exceptions.update(
{'__enable_maintenance_mode': traceback.format_exc()})
_print(F"Enable Nextcloud maintenance mode: {self.FAILED}")
return False
def __export_config(self) -> bool:
try:
os.system(
F"docker exec {self.app_container} tar -cf config.tar config/")
os.system(
F"docker cp {self.app_container}:/var/www/html/config.tar {self.__tar_file_path}"
)
os.system(F"docker exec {self.app_container} rm config.tar")
if os.path.isfile(self.__tar_file_path):
with tarfile.open(self.__tar_file_path, 'r') as tarball:
tarball.extractall(self.tmp_dir)
status = os.path.isdir(os.path.join(self.tmp_dir, "config"))
_print(
F"Export Nextcloud configuration: {self.SUCCESS if status else self.FAILED}"
)
return status
else:
_print(F"Export Nextcloud configuration: {self.FAILED}")
except:
_print(F"Export Nextcloud configuration: {self.FAILED}")
self.exceptions.update({'__export_config': traceback.format_exc()})
return False
else:
list_of_values.append({'value': val.get_text().strip()})
infobox[row.find('th').get_text(" ")] = list_of_values
return infobox
if __name__ == '__main__':
# load request store or create one
if not os.path.exists('data/request_store.pkl'):
request_store = {}
else:
with open('data/request_store.pkl', 'rb') as f:
request_store = pickle.load(f)
_print('Loaded request store')
# load movie info json
with open('data/wiki_movie_infos.json', 'rb') as f:
movie_list = json.load(f)
_print('Loaded movie info')
movie_metadata = {}
list_counter = 0
md_counter = 0
for country in movie_list:
for country_year in movie_list[country]:
for movie in movie_list[country][country_year]:
list_counter = list_counter + 1
print("Scrapping for {}".format(country_year))
try:
wiki_url = URL + movie['url']
except:
def restore():
# Set flags
utils.set_flags(sys.argv)
# Load config
config = Path(__file__).parent / "config.yml"
with open(config) as file:
# Load config
config_list = yaml.full_load(file)
log = Log(config_list['log']['log_dir'])
containers = Container.instantiate_containers(config_list)
# If any container names were passed as parameters, do only restore them
containers_wanted = {name: container for name, container in containers.items() if name in sys.argv}
if containers_wanted:
containers = containers_wanted
# If no container was chosen ask for it
elif not utils.all_containers:
containers_to_choose_from = [container.name for container in containers.values()]
terminal_menu = TerminalMenu(containers_to_choose_from, title="For which Nextcloud instance do you want "
"to restore a backup?")
choice_index = terminal_menu.show()
containers = {containers_to_choose_from[choice_index]: containers.get(containers_to_choose_from[choice_index])}
container: Container
for container in containers.values():
# Start restore
_print("----------------------------------------------")
_print(F"Restore backup for {container.name}")
backup_dir = os.scandir(container.backup_dir)
backup_files = {file.name: file for file in backup_dir if
file.is_file() and file.name.startswith(container.name) and file.name.endswith(".tar.gz")}
if len(backup_files) < 1:
_print(F"{Fore.YELLOW}No backups found for {container.name}{Style.RESET_ALL}")
break
backup_files_to_choose_from = [file.name for file in backup_files.values()]
backup_files_to_choose_from.sort(reverse=True)
_print()
# Choose backup to restore from
terminal_menu = TerminalMenu(backup_files_to_choose_from, title="Which backup do you want to restore?")
choice_index = terminal_menu.show()
backup_file = backup_files.get(backup_files_to_choose_from[choice_index])
print(backup_file.path)
# Confirm restore
if not utils.no_confirm:
confirm = input(F"Are you sure that you want to restore {backup_files_to_choose_from[choice_index]}? "
F"(Type: yes)\n").lower() == "yes"
else:
confirm = False
# Do the restore
if confirm or utils.no_confirm:
result = container.restore_backup(backup_file.path)
else:
break
# Print result and log
if result:
_print(F"{Fore.GREEN}Backup {container.restore_tar_file} for {container.name} successfully restored.{Style.RESET_ALL}")
if not utils.no_log and config_list['log']['logging']:
log.log(F"Restore backup ; {container.name} ; {container.restore_tar_file_path} ; SUCCESS")
else:
_print(F"{Fore.RED}Could not restore {container.restore_tar_file} for {container.name}.{Style.RESET_ALL}")
for func, traceback in container.exceptions.items():
_print()
_print(F"{Fore.YELLOW}Exception occurred in method: Container.{func}(){Style.RESET_ALL}")
_print(traceback)
_print()
if not utils.no_log and config_list['log']['logging']:
log.log(F"Restore backup ; {container.name} ; {container.restore_tar_file_path} ; FAIL")
def upgrade():
# Set flags
utils.set_flags(sys.argv)
# Load config
config = Path(__file__).parent / "config.yml"
with open(config) as file:
# Load config
config_list = yaml.full_load(file)
log = Log(config_list['log']['log_dir'])
containers = Container.instantiate_containers(config_list)
# If any container names were passed as parameters, do only upgrade them
containers_wanted = {
name: container
for name, container in containers.items() if name in sys.argv
}
if containers_wanted:
containers = containers_wanted
# If no container was chosen ask for it
elif not utils.all_containers:
containers_to_choose_from = [
container.name for container in containers.values()
]
terminal_menu = TerminalMenu(
containers_to_choose_from,
title="Which Nextcloud instance do you want to "
"upgrade?")
choice_index = terminal_menu.show()
containers = {
containers_to_choose_from[choice_index]:
containers.get(containers_to_choose_from[choice_index])
}
# Loop through Nextcloud container instances
container: Container
for container in containers.values():
go_on = True
# Make a backup
if not utils.no_backup:
utils.keep_maintenance_mode = True
go_on = backup.backup()
if go_on:
# Make the upgrade
utils.keep_maintenance_mode = True if "--maintenance" in sys.argv else False
_print("----------------------------------------------")
_print(
F"Start upgrade for {container.name} at {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
)
result = container.upgrade()
if result == 1:
_print(
F"{Fore.GREEN}{container.name} upgraded successfully{Style.RESET_ALL}"
)
upgrade_status = True
elif result == 2:
_print(
F"{Fore.GREEN}No upgrades available for {container.name}.{Style.RESET_ALL}"
)
upgrade_status = True
else:
_print(
F"{Fore.RED}Upgrade for {container.name} failed{Style.RESET_ALL}"
)
for func, traceback in container.exceptions.items():
_print()
_print(
F"{Fore.YELLOW}Exception occurred in method: Container.{func}(){Style.RESET_ALL}"
)
_print(traceback)
_print()
# Log upgrade
if not utils.no_log and config_list['log']['logging']:
if upgrade_status:
log.log(F"Upgrade ; {container.name} ; SUCCESS")
else:
log.log(F"Upgrade ; {container.name} ; FAIL")
if len(log.exceptions) > 0:
for func, traceback in log.exceptions.items():
_print()
_print(
F"{Fore.YELLOW}Exception occurred in method: Log.{func}(){Style.RESET_ALL}"
)
_print(traceback)
_print()
parts[temp[-1]] = map(lambda x: int(x), temp[0:-1])
fd.close()
return parts
def get_disk_partition():
partordisks = get_all_partitions().keys()
ret = {}
for i in partordisks:
temp_path = block_dir+'/'+i
if os.path.exists(temp_path):
#ret[i] = []
parts = [f for f in os.listdir(temp_path) if f in partordisks]
ret[i] = parts
return ret
def get_partition_disk():
diskparts = get_disk_partition()
ret = {}
for d, parts in diskparts.items():
for p in parts:
ret[p] = d
return ret
if __name__ == '__main__':
from utils import _print
_print(get_all_partitions())
_print(get_disk_partition())
_print(get_partition_disk())
def print_result(self):
print("total success:{}".format(len(self.success_list)))
print("total failed:{}, detail:".format(len(self.failed_list)))
for failed in self.failed_list:
_print(failed)