def startApplication(self):
"""
Connects to FTP-Server.
Checks state.
Downloads database.
Might throw exception.
Don't forget calling stopApplication before exiting the application.
:return:
"""
if self._ftp: # FTP connection already established
return
# get paths and filenames needed to establish a FTP connection:
self._ftpini = FtpIni(self._ftpini_pathnfile)
# initialize Ftp class and connect to FTP server
self._ftp = Ftp(self._ftpini)
self._ftp.connect()
# for comparing purposes download serverside immo_state
try:
serverstate: ImmoState = self._getServerState()
except Exception as ex:
raise Exception(
"IccStateHandler.startApplication():\nCan't get serverside state:\n%s"
% str(ex))
if serverstate.isInUse:
raise Exception(
"IccStateHandler.startApplication():\nCan't start application - Database is in use"
)
try:
localstate: ImmoState = self._getLocalState()
except Exception as ex:
raise Exception(
"IccStateHandler.startApplication():\nCan't get local state:\n%s"
% str(ex))
# compare serverside and locas last updates and raise exception if local last update is newer than
# serverside last update
if serverstate.lastUpdate < localstate.lastUpdate:
raise Exception(
"IccStateHandler.startApplication():\n"
"Expected serverside last update to be newer than local last update but found:\n"
"serverside = %s -- local = %s" %
(serverstate.lastUpdate, localstate.lastUpdate))
else:
# Everything is okay. We may download the serverside database now for local use.
try:
self._ftp.download(self.immo_dbname, self.immo_dbname)
except Exception as ex:
raise Exception("IccStateHandler.startApplication():\n"
"Download of serverside immo.db failed:\n%s" %
str(ex))
# Set is-in-use state in local and serverside immo_state files.
# Attribute last_update remains unchanged.
localstate.isInUse = True
try:
self._setStateAndSave(localstate)
except Exception as ex:
raise Exception(
"IccStateHandler.startApplication():\n"
"After download of serverside immo.db:\nFailed to set state:\n%s"
% str(ex))
class FtpRepository(Repository):
def __init__(self, ip, username, password, timeout):
self.connection = Ftp(ip, username, password, timeout)
def publish(self, name, raster_file):
with self.connection:
self.connection.create_directory_if_not_exists(name)
self.connection.upload_file(raster_file,
join(name, basename(raster_file)))
def append(msg):
today = datetime.datetime.today()
msg = str(today)[:-7] + " " + msg + "\r\n"
print(msg)
if os.path.isfile(Log.log_file_path):
file = open(Log.log_file_path, 'a+')
else:
file = open(Log.log_file_path, 'w+')
file.write(msg)
file.close()
Ftp.upload_file(Log.log_file_path, 'log.txt')
class FtpRepository(Repository):
def __init__(self, host, username, password, timeout, path=''):
self.path = path
self.connection = Ftp(host, username, password, timeout)
def publish(self, name, raster_file):
directory = join(self.path, name)
with self.connection:
self.connection.create_directory_if_not_exists(directory)
self.connection.upload_file(raster_file,
join(directory, basename(raster_file)))
def onRegister(self, response):
if response["body"] == 1:
QMessageBox.information(
self, "Error",
"user name already existed, please change another name!")
else:
# 注册成功，上传头像信息
QMessageBox.information(
self, "Congratulations",
"Register successfully, go back to login now!")
ftp = Ftp()
ftp.uploadfile(
"/home/meicorl/ChatServer/image/" +
os.path.basename(response["portrait"]), response["portrait"])
# 在本地创建用户目录
os.mkdir("user/" + response["name"])
def instauto(self):
while True:
status, cfg = self.read_status_cfg()
if not status:
logger.error("配置读取失败!")
else:
scheme = cfg.get("scheme")
if tupdate1.status == tupdate1.ST_INIT or tupdate1.status == tupdate1.ST_INSTALLFINISH or tupdate1.status == tupdate1.ST_INSTALLFAIL or tupdate1.ST_DOWNFAILED:
try:
if scheme == "http" or scheme == "https":
res = self.compare_versions()
r = json.loads(res)
state = r.get("success", "")
if state == True:
tupdate()
elif scheme == "ftp":
print "------->ftp updata start"
ftp_ip = cfg.get("netloc")
ftp_port = int(cfg.get("port"))
ftp_user = cfg.get("user")
ftp_pass = cfg.get("pass")
ftp_path = "/td01/update"
if not os.path.exists(ftp_path):
os.makedirs(ftp_path)
filename = ""
ftp = Ftp()
ftp.connect_ftp(ftp_ip, ftp_port, ftp_user,
ftp_pass)
for file in ftp.lst_file():
if os.path.splitext(file)[1] == '.pkt':
filename = file
break
else:
continue
filepath = ftp.downloadfile(ftp_path, filename)
ftp.delete(filename)
ftp.close()
self.packet_install(filepath)
print self.msg
except Exception as e:
logger.error(str(e))
time.sleep(self.starttime * 30)
def _exportDatabaseToServer( self ):
ftpini = FtpIni( "ftp.ini" )
ftp = Ftp( ftpini )
try:
ftp.connect()
ftp.upload( "immo.db", "immo.db" )
box = InfoBox( "Immo-Datenbak exportieren", "Exportieren abgeschlossen.", "", "OK" )
box.exec_()
except Exception as ex:
box = ErrorBox( "File Transfer failed", "Can't export immo.db to server:\n", str(ex) )
box.exec_()
finally:
ftp.quit()
def create_protocol():
protocol = os.environ.get("protocol")
logger.info("Using protocol: %s" % protocol)
if protocol.lower() == "ftp":
return Ftp()
elif protocol.lower() == "ssh":
return Ssh()
elif protocol.lower() == "json":
return Json()
else:
raise Exception("Unknown protocol: '%s'" % protocol)
class ProcessesGateway(object):
def __init__(self, host, user, password, timeout, directory):
self.connection = Ftp(host, user, password, timeout)
self.remote_directory = directory
def add(self, identifier, process_file_path):
with self.connection:
process_directory = self._get_remote_process_directory(identifier)
remote_file_path = join(process_directory,
basename(process_file_path))
self.connection.create_directory_if_not_exists(
self.remote_directory)
self.connection.create_directory_if_not_exists(process_directory)
self.connection.upload_file(process_file_path, remote_file_path)
def get(self, identifier, local_directory):
remote_process_dir = self._get_remote_process_directory(identifier)
local_process_dir = join(local_directory, identifier)
self._create_local_directory(local_process_dir)
with self.connection:
self.connection.download_directory_contents(
remote_process_dir, local_process_dir)
return local_process_dir
def remove(self, identifier):
with self.connection:
self.connection.delete_non_empty_directory(
self._get_remote_process_directory(identifier))
def get_process_context(self, identifier, local_directory):
local_directory = self.get(identifier, local_directory)
return ProcessContext(identifier, local_directory)
def _get_remote_process_directory(self, identifier):
return join(self.remote_directory, identifier)
@staticmethod
def _create_local_directory(directory):
if not exists(directory):
makedirs(directory)
def sync():
Ftp.download_file(State.state_server_file_path, 'state.json')
time.sleep(0.5)
server_state = State.load_state_from_server_file()
local_state = State.load_state_from_file()
local_state['pump_1_interval'] = server_state['pump_1_interval']
local_state['pump_2_interval'] = server_state['pump_2_interval']
local_state['pump_1_water_amount'] = server_state['pump_1_water_amount']
local_state['pump_2_water_amount'] = server_state['pump_2_water_amount']
local_state['tank_capacity'] = server_state['tank_capacity']
if server_state['tank_refilled'] == 1: # tank refilled
local_state['water_level'] = local_state['tank_capacity']
local_state['tank_refilled'] = 0
local_state['low_water_level_alert'] = 0
Log.append('tank refilled')
if server_state['run_test'] == 2: # test request
if local_state['run_test'] == 0: # request not taken
local_state['run_test'] = 2
elif local_state['run_test'] == 1: # request satisfied
local_state['run_test'] = 0
State.save_state_to_file(local_state)
Ftp.upload_file(State.state_file_path, 'state.json')
def updateFriendList(self, friendDict):
# 拉取好友头像
friend = json.loads(friendDict["friend_info"])
ftp = Ftp()
ftp.downloadfile("/home/meicorl/ChatServer/" + friend["portrait"],
friend["portrait"])
ftp.quit()
self.addFriend(friend["name"], friend["signature"], friend["portrait"])
# 将新添加的好友信息存入本地文件
file = open("user/" + self.userName.text() + "/friend_list.txt",
"a",
encoding="UTF-8")
file.write(json.dumps(friend))
file.close()
def uploadDatabase() -> bool:
"""
upload immo.db to server
:return: True if upload was successful
"""
ftpini = FtpIni("ftp.ini")
ftp = Ftp(ftpini)
try:
ftp.connect()
ftp.upload("immo.db", "immo.db")
return True
except Exception as ex:
box = ErrorBox("File Transfer failed", "FTP failed.\n", str(ex))
box.exec_()
return False
def _importDatabaseFromServer( self ):
ftpini = FtpIni( "ftp.ini" )
dlg = QInputDialog()
dlg.move( self._mainwin.cursor().pos() )
name, ok = dlg.getText( self._mainwin, "Immo-Datenbank importieren",
"Datenbank wird ins Verzeichnis\n\n'%s'\n\n importiert.\n\n"
"<<<<Sie wird nicht für die laufende Anwendung verwendet!!>>>>\n\n"
"Lokalen Namen für die Datenbank angeben: " % ftpini.getLocalPath(),
QLineEdit.Normal, "immo.db.imported" )
if not ( ok and name ): return
ftp = Ftp( ftpini )
try:
ftp.connect()
ftp.download( "immo.db", name )
box = InfoBox( "Immo-Datenbak importieren", "Importieren abgeschlossen.", "", "OK" )
box.move( self._mainwin.cursor().pos() )
box.exec_()
except Exception as ex:
box = ErrorBox( "File Transfer failed", "Can't export immo.db to server:\n", str( ex ) )
box.exec_()
finally:
ftp.quit()
def __login(self):
self.__setEnabled(True)
Room.user_name = self.userName.text()
FriendWind.user_name = self.userName.text()
# 更新窗口标题和Icon
home_path = "user/" + self.userName.text() + "/self_info.txt"
if not os.path.exists(home_path):
ftp = Ftp()
ftp.downloadfile(
"/home/meicorl/ChatServer/user/" + self.userName.text() +
"/self_info.txt", home_path)
ftp.quit()
file = open(home_path, encoding='UTF-8')
line = file.readline()
file.close()
self_info = json.loads(line)
self.setWindowTitle(self_info["name"] + "(" + self_info["signature"] +
")")
self.setWindowIcon(QIcon(self_info["portrait"]))
# 初始化好友列表
self.list.setEnabled(True)
self.__InitFriendList()
def __InitFriendList(self):
# 初始化好友列表
self.list.clear()
self.friendList.clear()
friendListPath = "./user/" + self.userName.text() + "/friend_list.txt"
# 如果本地好友列表丢失，则从服务器拉取好友列表
if not os.path.exists(friendListPath):
ftp = Ftp()
ftp.downloadfile(
"/home/meicorl/ChatServer/user/" + self.userName.text() +
"/friend_list.txt", friendListPath)
ftp.quit()
if os.path.exists(friendListPath):
file = open(friendListPath, encoding='UTF-8')
while 1:
line = file.readline()
if not line:
break
friend = json.loads(line)
self.addFriend(friend["name"], friend["signature"],
friend["portrait"])
file.close()
import os
sys.path.insert(1, '/var/www/html/worldfone4xs_ibm/cronjob/python')
from ftp import Ftp
from pprint import pprint
from mongod import Mongodb
from excel import Excel
from datetime import datetime
from datetime import date
from bson import ObjectId
try:
filename = 'ZACCF full.csv'
mongodb = Mongodb("worldfone4xs")
_mongodb = Mongodb("_worldfone4xs")
excel = Excel()
ftp = Ftp()
now = datetime.now()
sibsColumns = []
sibsConverters = {}
ftp.connect()
ftp.downLoadFile("/var/www/html/worldfone4xs_ibm/upload/csv/ftp/" + filename, filename)
ftp.close()
path, filename = os.path.split("/var/www/html/worldfone4xs_ibm/upload/csv/ftp/" + filename)
importLogInfo = {
'collection' : "Sibs",
'begin_import' : time.time(),
'file_name' : filename,
from ftp import Ftp
host = input("Please Enter Host\n")
username = input("Please Enter Username\n")
password = input("Please enter password\n")
ftp = Ftp(host,username,password)
while (True):
mode = int(input("1.Download \n2.Upload\n"))
if (mode == 1):
ftp.listDirectory()
filename = input("Please enter path of file to download\n")
ftp.downloadFile(filename)
else:
filename = input("Please enter path file to upload\n")
ftp.uploadFile(filename)
def __init__(self, host, user, password, timeout, directory):
self.connection = Ftp(host, user, password, timeout)
self.remote_directory = directory
class IccStateHandler:
"""
IccFileTransfer handles all ftp up- and downloads concerning the ImmoControlCenter application.
- beauskunftet den Status der immo-Datenbanken
a) auf dem Server
b) lokal
using file immo_state.
Uploads and downloads immo_state and immo.db.
Structure of file immo_state:
state=not_in_use
last_update=2021-04-22
- processes up- and donwloads as needed
"""
attributename_state = "state"
attributename_lastupdate = "last_update"
state_in_use = "in_use"
state_not_in_use = "not_in_use"
state_last_update_unknown = "unknown"
immostate_sep = "="
def __init__(self, ftpini_pathnfile: str):
self._ftpini_pathnfile = ftpini_pathnfile
self.immostate_filename = "immo_state" # remote and local filename
self.immostate_localfilename_tmp = "immo_state.tmp" # local name after download
self.immo_dbname = "immo.db" # remote and local filename
self._ftp: Ftp = None
self._ftpini: FtpIni = None
self.currentState: ImmoState = None
def startApplication(self):
"""
Connects to FTP-Server.
Checks state.
Downloads database.
Might throw exception.
Don't forget calling stopApplication before exiting the application.
:return:
"""
if self._ftp: # FTP connection already established
return
# get paths and filenames needed to establish a FTP connection:
self._ftpini = FtpIni(self._ftpini_pathnfile)
# initialize Ftp class and connect to FTP server
self._ftp = Ftp(self._ftpini)
self._ftp.connect()
# for comparing purposes download serverside immo_state
try:
serverstate: ImmoState = self._getServerState()
except Exception as ex:
raise Exception(
"IccStateHandler.startApplication():\nCan't get serverside state:\n%s"
% str(ex))
if serverstate.isInUse:
raise Exception(
"IccStateHandler.startApplication():\nCan't start application - Database is in use"
)
try:
localstate: ImmoState = self._getLocalState()
except Exception as ex:
raise Exception(
"IccStateHandler.startApplication():\nCan't get local state:\n%s"
% str(ex))
# compare serverside and locas last updates and raise exception if local last update is newer than
# serverside last update
if serverstate.lastUpdate < localstate.lastUpdate:
raise Exception(
"IccStateHandler.startApplication():\n"
"Expected serverside last update to be newer than local last update but found:\n"
"serverside = %s -- local = %s" %
(serverstate.lastUpdate, localstate.lastUpdate))
else:
# Everything is okay. We may download the serverside database now for local use.
try:
self._ftp.download(self.immo_dbname, self.immo_dbname)
except Exception as ex:
raise Exception("IccStateHandler.startApplication():\n"
"Download of serverside immo.db failed:\n%s" %
str(ex))
# Set is-in-use state in local and serverside immo_state files.
# Attribute last_update remains unchanged.
localstate.isInUse = True
try:
self._setStateAndSave(localstate)
except Exception as ex:
raise Exception(
"IccStateHandler.startApplication():\n"
"After download of serverside immo.db:\nFailed to set state:\n%s"
% str(ex))
def stopApplication(self):
"""
Uploads database
Sets state and last_update attributes in immo_state.
Uploads immo_state
Disconnects from FTP-Server
Might throw exception.
:return:
"""
try:
self._ftp.upload(self.immo_dbname, self.immo_dbname)
except Exception as ex:
raise Exception("IccStateHandler.stopApplication():\n"
"Upload immo.db failed:\n%s" % str(ex))
state = ImmoState()
state.isInUse = False
state.lastUpdate = datehelper.getCurrentTimestampIso()
try:
self._setStateAndSave(state)
except Exception as ex:
raise Exception(
"IccStateHandler.stopApplication():\n"
"After upload of local immo.db:\nFailed to set state:\n%s" %
str(ex))
self._ftp.quit()
self._ftp = None
def _getServerState(self) -> ImmoState:
self._ftp.download(self.immostate_filename,
self.immostate_localfilename_tmp)
localpathnfile_tmp = self._ftpini.getLocalPath(
) + self.immostate_localfilename_tmp
serverstate: ImmoState = self._readAttributesFromState(
localpathnfile_tmp)
return serverstate
def _setStateAndSave(self, immoState: ImmoState) -> None:
state = self.state_in_use if immoState.isInUse else self.state_not_in_use
last_update = immoState.lastUpdate
stateline = self.attributename_state + self.immostate_sep + state
lastupdateline = self.attributename_lastupdate + self.immostate_sep + last_update
self._setLocalAndServerStateAndSave([stateline, "\n", lastupdateline])
def _setLocalAndServerStateAndSave(self, lines: list) -> None:
"""
:param lines: attribute names and values to write into immo_state
:return:
"""
pathnfile = self._ftpini.getLocalPath() + self.immostate_filename
try:
with open(pathnfile, "w") as statefile:
for line in lines:
statefile.write(line)
except Exception as ex:
raise Exception(
"IccStateHandler._setLocalAndServerStateAndSave():\n"
"Local storage of file immo_state failed.\n%s" % str(ex))
try:
self._ftp.upload(self.immostate_filename, self.immostate_filename)
except Exception as ex:
raise Exception(
"IccStateHandler._setLocalAndServerStateAndSave():\n"
"Serverside storage of file immo_state failed.\n%s" % str(ex))
def _getLocalState(self) -> ImmoState:
"""
gets state attributes from local immo_state file
:return:
"""
localpathnfile = self._ftpini.getLocalPath() + self.immostate_filename
localstate: ImmoState = self._readAttributesFromState(localpathnfile)
return localstate
def _readAttributesFromState(self, pathnfile: str) -> ImmoState:
"""
provides informations "database last update" and "database is in use" from file <pathnfile>
:param pathnfile: immo_state file server or local side in the prescribed structure (see class comment)
:return: an ImmoState object containing the attributes found in <pathnfile>
"""
immostate = ImmoState()
serverstatefile = open(pathnfile, "r")
content = serverstatefile.read()
serverstatefile.close()
lines = content.splitlines()
for line in lines:
parts = line.split(
self.immostate_sep) # list like so: ['state', 'not_in_use']
if len(parts) != 2:
raise Exception(
"IccFileTransfer._readAttributesFromState: Attribute invalid\n"
"Expected <attribute_name>=<attribute_value> -- found: %s"
% parts)
if parts[0] == self.attributename_state:
immostate.isInUse = True if parts[
1] == self.state_in_use else False
elif parts[0] == self.attributename_lastupdate:
immostate.lastUpdate = parts[1]
return immostate
def upload(path):
return Ftp.load().upload(to=path)
def download(path) -> Generator:
return Ftp.load().download_all(to=path)
def backup(config, logger=None):
"""
@param config : an array with the configuration
"""
backup_dir = datetime.now().isoformat(' ')
_backup_size = backup_size(config)
if _backup_size > config.get('max_size'):
raise Exception("The backup has a greater size than allowed !!")
# informations
user = config.get('server:user')
password = config.get('server:password')
# Connect to the FTP server backup
ftp = Ftp(config.get('server:host'))
ftp.login(user, password)
root = config.get('server:backup_root')
if root != '/' and not ftp.exists(root):
ftp.mkdirs(root)
#ftp.clear(root)
ftp.cwd(root)
while max_versions_reached(ftp, config):
delete_oldest_version(ftp, config, logger)
while backups_total_size(ftp, root, config) + _backup_size > config.get('max_size'):
delete_oldest_version(ftp, config, logger)
# create the backup directory and go to it
ftp.mkd(backup_dir)
ftp.cwd(backup_dir)
#upload bk_file
f = tempfile.TemporaryFile(mode='w+b')
f.write("%d" % (_backup_size))
f.seek(0)
ftp.storbinary("STOR %s" % (config.get('backup_size_file')), f)
f.close()
# Get file or dirs to upload
files = config.get('files')
# Get separators
server_separator = config.get('server:separator')
local_separator = os.sep
for f in files:
local_file = f['path']
logger.info("Save %s" % (local_file))
# ensure parent directory exists
local_parent, filename = os.path.split(local_file)
ftp.mkdirs(local_parent, server_separator)
# remote parent is the same as local parent but without the beginning slash
remote_parent = ftp.normpath(local_separator.join((root, backup_dir, local_parent[1:])))
tar_files = config.has('default:tar') and config.get('default:tar') == True
if 'tar' in f:
tar_files = (f['tar'] == True)
if tar_files:
tmp_dir = tempfile.gettempdir()
tar_filename = "%s.tar" % (filename)
tar_path = os.path.join(tmp_dir, tar_filename)
logger.debug("Create Tar file in temp directory : %s" % (tar_path))
tar = tarfile.open(tar_path, "w")
tar.add(os.path.join(local_parent, filename))
tar.close()
#for info in tar:
#print info
upload(ftp, tmp_dir, remote_parent, tar_filename, separator=server_separator, logger=logger)
logger.debug("Remove %s" % (tar_path))
os.remove(tar_path)
else:
# upload files
upload(ftp, local_parent, remote_parent, filename, separator=server_separator, logger=logger)
# close the connection
ftp.close()
from ftp import Ftp
import traceback
import datetime
ftp = Ftp('192.168.0.1')
ftp.login('admin', 'zdd2093419')
today = datetime.date.today()
remote_file = '/sda/91/' + str(today) + '/'
try:
ftp.mkd(remote_file)
except:
traceback.print_exc()
ftp.close()
def __init__(self, ip, username, password, timeout):
self.connection = Ftp(ip, username, password, timeout)
def __init__(self, host, username, password, timeout, path=''):
self.path = path
self.connection = Ftp(host, username, password, timeout)
def go(self):
html = self.__get_content(1)
#print(html)
page_num = self.__get_page_num(html)
print('页数=' + str(page_num))
today = datetime.date.today()
total_num = 0
for index in range(0, page_num):
result_all = []
html = self.__get_content(index + 1)
result = self.__analysis(html)
result_all.extend(result)
#print(result_all)
result_all = self.__refine(result_all)
#print(result_all)
result_all = self.__sort(result_all)
self.show(result_all)
result_all = self.__filter(result_all)
self.show(result_all)
print(str(today) + "有 " + str(len(result_all)) + " 部待上传")
if len(result_all) == 0:
continue
dir = Spider.PATH + str(today) + "\\"
if not os.path.exists(dir):
os.makedirs(dir)
ftp = Ftp('192.168.0.1')
ftp.login('admin', 'zdd2093419')
remote_file = '/KINGSTON-b/91/' + str(today) + '/'
try:
ftp.mkd(remote_file)
except:
traceback.print_exc()
ftp.close()
for index in range(0, len(result_all)):
video = result_all[index]
download_file = DownloadFile(dir, video['title'] + '.m3u8',
video['source_url'])
download_file.download_progress()
# 根据m3u8文件获取到所有ts的链接
result = spider.m3u8file(
re.findall(Spider.path_pattern, video['source_url'])[0],
dir + video['title'] + '.m3u8')
print(result)
all_ts_name = ''
# 下载所有ts
os.system("del " + dir + "*.ts")
for ts in result:
all_ts_name += dir + ts['file_name'] + '|'
download_file = DownloadFile(dir, ts['file_name'],
ts['sour_url'])
download_file.download_progress()
# 合并ts
os.system('ffmpeg -i "concat:' + all_ts_name.strip('|') +
'" -acodec copy -vcodec copy -absf aac_adtstoasc ' +
dir + video['title'] + '.mp4')
os.system("del " + dir + "*.ts")
# 下载完一个就上传一个
ftp = Ftp('192.168.0.1')
ftp.login('admin', 'zdd2093419')
try:
ftp.upload_file(dir + video['title'] + '.mp4',
remote_file + video['title'] + '.mp4')
Spider.mongo_collection.insert_one(video)
print("============ " + " 已上传 " + str(index + 1) +
" 部 ============")
total_num += 1
except:
traceback.print_exc()
ftp.close()
print("============ " + str(today) + " 总共上传 " + str(total_num) +
" 部 ============")