def process_ExternalHd(self, dirFTP, dirLOCAL):
print('Start backup %s to ExternalHd..............................................................................' %dirFTP)
localpath = self.mainExternalhd+'/'+dirLOCAL+'/'
localfiles = os.listdir(localpath)
with reconnecting_ftp.Client(self.hostname, self.port, self.user, self.password) as ftp:
ftp.cwd(dirFTP)
FTPcurrentDir = ftp.pwd()
FTPfiles = ftp.mlsd(FTPcurrentDir) #list files in ftp path
for i in FTPfiles:
FTPfileName = i[0]
ftpMtime = parser.parse(i[1]['modify'])
if not os.path.exists(self.mainExternalhd+'/'+dirLOCAL+'/'+ FTPfileName):
print ('Downloading to ExternalHd %s - Modified %s' %(FTPfileName, ftpMtime))
ftp.retrbinary('RETR %s'%(FTPcurrentDir+'/'+FTPfileName), open(self.mainExternalhd+'/'+dirLOCAL+'/'+FTPfileName, 'wb').write)
time.sleep(5)
else:
print('%s Already in LOCAL - Checking modified' %FTPfileName)
for localname in localfiles:
if fnmatch.fnmatch(localname, FTPfileName) is True:
localMtime = datetime.strptime(time.ctime(os.path.getmtime(localpath+localname)), '%a %b %d %H:%M:%S %Y')
#=======Checking modified data, Time is UTC=======
# print('FTP: %-5s %s %s %s' %(' ', FTPfileName, ftpMtime, type(ftpMtime)))
# print('Local: %-5s %s %s %s' %(' ', localname, localMtime, type(localMtime)))
if ftpMtime > localMtime:
print('FTP data was modified...Sending %s to Local backup' %FTPfileName)
ftp.retrbinary('RETR %s'%(FTPcurrentDir+'/'+FTPfileName), open(self.mainExternalhd+'/'+dirLOCAL+'/'+FTPfileName, 'wb').write)
time.sleep(5)
else:
print('%s Data on FTP server was not modified' %FTPfileName)
print('ExternalHd is checking back to FTP server.............................................................................')
localpath = self.mainExternalhd+'/'+dirLOCAL+'/'
localfiles = os.listdir(localpath)
# print(localfiles, type(localfiles))
with reconnecting_ftp.Client(self.hostname, self.port, self.user, self.password) as ftp:
ftp.cwd(dirFTP)
FTPcurrentDir = ftp.pwd()
FTPfiles = ftp.mlsd(FTPcurrentDir) #list files in ftp path
listFtp_FileName = []
for file in FTPfiles:
FTPfileName = file[0]
listFtp_FileName.append(FTPfileName)
# print(listFtp_FileName, type(listFtp_FileName))
diff = list(set(localfiles) - set(listFtp_FileName))
print("%s is not appear on FTP server" %diff)
if diff != []:
for file in diff:
os.remove(self.mainExternalhd+'/'+dirLOCAL+'/'+file)
print("{} was removed from ExternalHd".format(file))
else:
print("Nothings change")
def download(username, password, product, indate, outdate):
try:
with reconnecting_ftp.Client(hostname="ftphsaf.meteoam.it",
port=21,
user=username,
password=password) as ftp:
print('Connected')
dir = product + '/' + product + '_cur_mon_data'
ftp.cwd(dir)
in_ = indate
last_ = outdate
init_date = datetime.datetime.strptime(in_, "%Y%m%d")
last_date = datetime.datetime.strptime(last_, "%Y%m%d")
filelist = []
days = last_date - init_date
if not os.path.exists(product + '_data'):
os.makedirs(product + '_data')
for i in progressbar(range(days.days), "Downloading: ", 40):
date = ((init_date +
datetime.timedelta(days=i)).strftime("%Y%m%d"))
grabFile(product, date, ftp)
print("Download Process has been finished")
except:
print("username or password is not correct")
def test_mlsd(self):
with TestContext(port=find_free_port(), timeout=1) as ctx:
ftp = reconnecting_ftp.Client(hostname=ctx.hostname,
port=ctx.port,
user=ctx.user,
password=ctx.password)
with ftp:
pth = ctx.homedir / 'some-dir/some-file.txt'
pth.parent.mkdir(parents=True)
pth.write_text('tested')
ftp.cwd(dirname='/some-dir')
names = [] # type: List[str]
entry_dicts = [] # type: List[Dict[str, Any]]
for name, entry_dict in ftp.mlsd(facts=['size']):
names.append(name)
entry_dicts.append(entry_dict)
self.assertListEqual(names, ['some-file.txt'])
self.assertEqual(len(entry_dicts), 1)
self.assertDictEqual(entry_dicts[0], {'size': '6'})
def run(params: Params) -> None:
"""
Runs the webcam_ftpry.
:param params: parsed command-line parameters
:return:
"""
ftp = reconnecting_ftp.Client(
hostname=params.hostname, port=params.port, user=params.user, password=params.password)
with contextlib.ExitStack() as exit_stack:
if params.operation_dir:
operation_dir = pathlib.Path(params.operation_dir)
operation_dir.mkdir(exist_ok=True, parents=True)
else:
tmp_dir = tempfile.TemporaryDirectory()
exit_stack.push(tmp_dir)
operation_dir = pathlib.Path(tmp_dir.name)
cap = cv2.VideoCapture(params.device_id)
exit_stack.callback(callback=cap.release)
ftp.__enter__()
exit_stack.push(ftp)
logger = logging.getLogger('webcam_ftpry')
latest_upload = None # type: Optional[datetime.datetime]
while True:
ret, frame = cap.read()
if not ret:
raise RuntimeError("Unexpected end of video")
now = datetime.datetime.utcnow()
if latest_upload is None or (now - latest_upload).total_seconds() > params.period:
remote_pth = pathlib.Path(now.strftime(params.path_format))
with tempfile.NamedTemporaryFile(
prefix=remote_pth.stem, suffix=remote_pth.suffix, dir=operation_dir.as_posix()) as tmp_file:
local_pth = pathlib.Path(tmp_file.name)
if params.angle:
frame = rotate(image=frame, angle=params.angle)
ret = cv2.imwrite(local_pth.as_posix(), frame)
if not ret:
raise RuntimeError("Failed to save the image to: {}".format(local_pth))
msg = "Uploading the image from local {} to remote {}...".format(local_pth, remote_pth)
logger.log(logging.INFO, msg)
upload(ftp=ftp, local_path=local_pth, remote_path=remote_pth)
latest_upload = now
msg = "Image uploaded to {}".format(remote_pth)
logger.log(logging.INFO, msg)
def __enter__(self):
#_ftp = ftplib.FTP()
#_ftp.encoding = 'cp1252'
#_ftp.set_pasv(True)
self.ftp = reconnecting_ftp.Client(self.host, 7777, self.user,
self.pwd)
self.ftp.makepasv()
return self
def test_connect(self):
# pylint: disable=no-self-use
with TestContext(port=find_free_port(), timeout=1) as ctx:
ftp = reconnecting_ftp.Client(hostname=ctx.hostname,
port=ctx.port,
user=ctx.user,
password=ctx.password)
with ftp:
ftp.connect()
def test_timeout(self):
with TestContext(port=find_free_port(), timeout=1) as ctx:
ftp = reconnecting_ftp.Client(hostname=ctx.hostname,
port=ctx.port,
user=ctx.user,
password=ctx.password)
with ftp:
ftp.connect()
(ctx.homedir / 'some-dir/some-subdir').mkdir(parents=True)
ftp.cwd(dirname='/some-dir/some-subdir')
time.sleep(ctx.timeout +
1) # ensure we timed out and we can still reconnect
pth = ftp.pwd()
self.assertEqual(pth, '/some-dir/some-subdir')
def test_mlst(self):
with TestContext(port=find_free_port(), timeout=1) as ctx:
ftp = reconnecting_ftp.Client(hostname=ctx.hostname,
port=ctx.port,
user=ctx.user,
password=ctx.password)
with ftp:
pth = ctx.homedir / 'some-dir/some-file.txt'
pth.parent.mkdir(parents=True)
pth.write_text('tested')
ftp.cwd(dirname='/some-dir')
srv_pth, entry = ftp.mlst(filename='some-file.txt')
self.assertEqual(srv_pth, '/some-dir/some-file.txt')
modify = datetime.datetime.strptime(entry['modify'],
'%Y%m%d%H%M%S')
expected_modify = datetime.datetime.utcfromtimestamp(
pathlib.Path(pth).stat().st_mtime).replace(microsecond=0)
self.assertEqual(modify, expected_modify)
self.assertEqual(entry['type'], 'file')
with self.assertRaises(ftplib.error_perm):
_, _ = ftp.mlst(filename='/non-existing/some-file.txt')
ftp.cwd(dirname='/')
srv_pth, entry = ftp.mlst(filename='some-dir')
self.assertEqual(srv_pth, '/some-dir')
modify = datetime.datetime.strptime(entry['modify'],
'%Y%m%d%H%M%S')
expected_modify = datetime.datetime.utcfromtimestamp(
pathlib.Path(pth).stat().st_mtime).replace(microsecond=0)
self.assertEqual(modify, expected_modify)
self.assertEqual(entry['type'], 'dir')