def reconnect(self):
self._connection = ftplib.FTP(self.host,
self.user,
self.password,
timeout=5)
timeout = 1800
socket.setdefaulttimeout(timeout)
# FTP: ftp://ftp.ncbi.nlm.nih.gov/genomes/Viruses/Enterobacteria_phage_phiX174_sensu_lato_uid14015/NC_001422.fna
# web: http://www.ncbi.nlm.nih.gov/nuccore/9626372
url = 'genomes/Viruses/Enterobacteria_phage_phiX174_sensu_lato_uid14015/'
version = ""
#mypath = os.path.join(options.output_directory,'phix174_index')
mypath = options.output_directory
if not os.path.isdir(mypath):
os.makedirs(mypath)
print "Downloading the 'Enterobacteria phage phiX174' genome from NCBI server!"
try:
ftp = ftplib.FTP(options.server)
print ftp.login()
ftp.cwd(url)
list_files = ftp.nlst()
list_files = [el for el in list_files if el.lower() == 'nc_001422.fna']
if len(list_files) == 1:
new_files = []
for filename in list_files:
print "Downloading: %s/%s/%s" % (options.server,url,filename)
nf = os.path.join(mypath,filename)
new_files.append(nf)
fid = open(nf,'wb')
ftp.retrbinary("RETR " + filename, fid.write)
class myFtp:
ftp = ftplib.FTP()
bIsDir = False
path = ""
def connetc(self, host='', port=21, timeout=-999):
self.ftp.connect(self, host, port, timeout)
def getwelcome(self):
self.ftp.getwelcome()
def cwd(self, dirname):
res = self.ftp.cwd(dirname)
return res
def dir(self, *args):
self.ftp.dir(*args)
def __init__(self, host, port='21'):
# self.ftp.set_debuglevel(2) # 打开调试级别2，显示详细信息
# self.ftp.set_pasv(0) # 0主动模式 1 #被动模式
self.ftp.connect(host, port)
def Login(self, user, passwd):
self.ftp.login(user, passwd)
logging.info(self.ftp.welcome)
def DownLoadFile(self, LocalFile, RemoteFile):
file_handler = open(LocalFile, 'wb')
self.ftp.retrbinary("RETR %s" % (RemoteFile), file_handler.write)
file_handler.close()
return True
def UpLoadFile(self, LocalFile, RemoteFile):
if os.path.isfile(LocalFile) is False:
return False
file_handler = open(LocalFile, "rb")
self.ftp.storbinary('STOR %s' % RemoteFile, file_handler, 4096)
file_handler.close()
return True
def UpLoadFileTree(self, LocalDir, RemoteDir):
if os.path.isdir(LocalDir) is False:
return False
logging.info("LocalDir: %s" % LocalDir)
LocalNames = os.listdir(LocalDir)
print "list:", LocalNames
print RemoteDir
self.ftp.cwd(RemoteDir)
for Local in LocalNames:
src = os.path.join(LocalDir, Local)
if os.path.isdir(src):
self.UpLoadFileTree(src, Local)
else:
self.UpLoadFile(src, Local)
self.ftp.cwd("..")
return
def DownLoadFileTree(self, LocalDir, RemoteDir):
print "remoteDir:", RemoteDir
if os.path.isdir(LocalDir) is False:
os.makedirs(LocalDir)
self.ftp.cwd(RemoteDir)
RemoteNames = self.ftp.nlst()
print "RemoteNames", RemoteNames
print self.ftp.nlst("/del1")
for filee in RemoteNames:
Local = os.path.join(LocalDir, filee)
if self.isDir(filee):
self.DownLoadFileTree(Local, filee)
else:
self.DownLoadFile(Local, filee)
self.ftp.cwd("..")
return
def show(self, listt):
result = listt.lower().split(" ")
if self.path in result and "<dir>" in result:
self.bIsDir = True
def isDir(self, path):
self.bIsDir = False
self.path = path
# this ues callback function ,that will change bIsDir value
self.ftp.retrlines('LIST', self.show)
return self.bIsDir
def retrlines(self, cmd, callback=None):
res = self.ftp.retrlines(cmd, callback)
return res
def mkd(self, dirname):
resStr = self.ftp.mkd(dirname)
return resStr
def rmd(self, dirname):
resStr = self.ftp.rmd(dirname)
return resStr
def delete(self, filename):
resStr = self.ftp.delete(filename)
return resStr
def pwd(self):
resStr = self.ftp.pwd()
return resStr
def nlst(self):
resStr = self.ftp.nlst()
return resStr
def close(self):
self.ftp.quit()
def _get_IONEX_file(
time="2012/03/23/02:20:10.01",
server="ftp://cddis.gsfc.nasa.gov/gnss/products/ionex/",
prefix="codg",
outpath='./',
overwrite=False,
backupserver="ftp://cddis.gsfc.nasa.gov/gnss/products/ionex/"):
"""Get IONEX file with prefix from server for a given day
Downloads files with given prefix from the ftp server, unzips and stores
the data. For prefix ROBR the data is stored on the server in a separate
IONEX file every 15 minutes, these are automatically combined for
compatibility.
Args:
time (string or list) : date of the observation
server (string) : ftp server + path to the ionex directories
prefix (string) : prefix of the IONEX files (case insensitive)
outpath (string) : path where the data is stored
overwrite (bool) : Do (not) overwrite existing data
"""
prefix = prefix.upper()
if outpath[-1] != "/":
outpath += "/"
if not os.path.isdir(outpath):
try:
os.makedirs(outpath)
except:
logging.error("cannot create output dir for IONEXdata: %s",
outpath)
try:
yy = time[2:4]
year = int(time[:4])
month = int(time[5:7])
day = int(time[8:10])
except:
year = time[0]
yy = year - 2000
month = time[1]
day = time[2]
mydate = datetime.date(year, month, day)
dayofyear = mydate.timetuple().tm_yday
#if file exists just return filename
if not overwrite and os.path.isfile("%s%s%03d0.%02dI" %
(outpath, prefix, dayofyear, yy)):
logging.info("FILE exists: %s%s%03d0.%02dI", outpath, prefix,
dayofyear, yy)
return "%s%s%03d0.%02dI" % (outpath, prefix, dayofyear, yy)
#check if IGRG (fast files) exist, use those instead (UGLY!!)
if not overwrite and os.path.isfile("%sIGRG%03d0.%02dI" %
(outpath, dayofyear, yy)):
logging.info("fast FILE exists: %sIGRG%03d0.%02dI", outpath, dayofyear,
yy)
return "%sIGRG%03d0.%02dI" % (outpath, dayofyear, yy)
tried_backup = False
serverfound = False
while not serverfound:
ftpserver = server.replace("ftp:", "").strip("/").split("/")[0]
ftppath = "/".join(
server.replace("ftp:", "").strip("/").split("/")[1:])
nr_tries = 0
try_again = True
while try_again and nr_tries < 10:
try:
ftp = ftplib.FTP(ftpserver)
ftp.login()
try_again = False
serverfound = True
except ftplib.error_perm:
if "213.184.6.172" in server:
ftp.login("data-out", "Qz8803#mhR4z")
try_again = False
serverfound = True
else:
try_again = True
nr_tries += 1
if nr_tries >= 10:
if tried_backup or server == backupserver:
raise Exception("Could not connect to %s" %
ftpserver)
else:
server = backupserver
tried_backup = True
except socket.gaierror:
try_again = True
nr_tries += 1
if nr_tries >= 10:
if tried_backup or server == backupserver:
raise Exception("Could not connect to %s" % ftpserver)
else:
server = backupserver
tried_backup = True
ftp.cwd(ftppath)
totpath = ftppath
myl = []
ftp.retrlines("NLST", myl.append)
logging.info("Retrieving data for %d or %02d%03d", year, yy, dayofyear)
if str(year) in myl:
ftp.cwd(str(year))
totpath += "/%d" % (year)
elif "%02d%03d" % (yy, dayofyear) in myl:
ftp.cwd("%02d%03d" % (yy, dayofyear))
totpath += "/%02d%03d" % (yy, dayofyear)
myl = []
ftp.retrlines("NLST", myl.append)
if "%03d" % dayofyear in myl:
ftp.cwd("%03d" % dayofyear)
totpath += "/%03d" % (dayofyear)
logging.info("Retrieving data from %s", totpath)
myl = []
ftp.retrlines("NLST", myl.append)
filenames = [
i for i in myl
if (prefix.lower() in i.lower()) and ("%03d" % dayofyear in i.lower())
and (i.lower().endswith("i.z") or i.lower().endswith("i"))
]
logging.info(" ".join(filenames))
#assert len(filenames) > 0, "No files found on %s for %s" % (server,prefix)
if len(filenames) <= 0:
logging.info("No files found on %s for %s", server, prefix)
return -1
if prefix.lower() == "robr" and len(filenames) > 1:
filenames = sorted(filenames)
filenames = _store_files(ftp, filenames, outpath, overwrite)
# get data for next day
nextday = mydate + datetime.timedelta(days=1)
nyear = nextday.year
ndayofyear = nextday.timetuple().tm_yday
ftp.cwd("/" + ftppath)
myl = []
ftp.retrlines("NLST", myl.append)
if str(nyear) in myl:
ftp.cwd(str(nyear))
myl = ftp.retrlines("NLST")
if str(ndayofyear) in myl:
ftp.cwd(str(ndayofyear))
myl = ftp.retrlines("NLST")
nfilenames = [
i for i in myl if (prefix.lower() in i.lower()) and (
i.lower().endswith("i.z")) and "A00" in i.upper()
]
nfilenames = _store_files(ftp, nfilenames, outpath, overwrite)
filenames += nfilenames
_combine_ionex(outpath, filenames,
prefix + "%03d0.%sI" % (dayofyear, yy))
ftp.quit()
return os.path.join(outpath, prefix + "%03d0.%sI" % (dayofyear, yy))
else:
nfilenames = _store_files(ftp, filenames, outpath, overwrite)
ftp.quit()
return nfilenames[0]
def upload_ftp(self, url=None, directory=None, user=None, password=None):
"""Attempts to upload the archive to either the policy defined or user
provided FTP location.
:param url: The URL to upload to
:type url: ``str``
:param directory: The directory on the FTP server to write to
:type directory: ``str`` or ``None``
:param user: The user to authenticate with
:type user: ``str``
:param password: The password to use for `user`
:type password: ``str``
:returns: ``True`` if upload is successful
:rtype: ``bool``
:raises: ``Exception`` if upload in unsuccessful
"""
try:
import ftplib
import socket
except ImportError:
# socket is part of the standard library, should only fail here on
# ftplib
raise Exception("missing python ftplib library")
if not url:
url = self.get_upload_url()
if url is None:
raise Exception("no FTP server specified by policy, use --upload-"
"url to specify a location")
url = url.replace('ftp://', '')
if not user:
user = self.get_upload_user()
if not password:
password = self.get_upload_password()
if not directory:
directory = self.upload_directory or self._upload_directory
try:
session = ftplib.FTP(url, user, password, timeout=15)
if not session:
raise Exception("connection failed, did you set a user and "
"password?")
session.cwd(directory)
except socket.timeout:
raise Exception("timeout hit while connecting to %s" % url)
except socket.gaierror:
raise Exception("unable to connect to %s" % url)
except ftplib.error_perm as err:
errno = str(err).split()[0]
if errno == '503':
raise Exception("could not login as '%s'" % user)
if errno == '530':
raise Exception("invalid password for user '%s'" % user)
if errno == '550':
raise Exception("could not set upload directory to %s" %
directory)
raise Exception("error trying to establish session: %s" % str(err))
try:
with open(self.upload_archive_name, 'rb') as _arcfile:
session.storbinary(
"STOR %s" % self.upload_archive_name.split('/')[-1],
_arcfile)
session.quit()
return True
except IOError:
raise Exception("could not open archive file")
path = f'/home/backup/{day}/mikrotik'
if not os.path.isdir(path):
os.makedirs(path)
os.chdir(path)
filename_pattern = '{}_backup.rsc'
for device in devices:
print("Connect to {}:".format(device['ip']))
filename = filename_pattern.format(device['name'])
try:
with ftplib.FTP(device['ip'], device['user_name'],
device['users_passwd']) as con:
with open(filename, "wb") as f:
con.retrbinary('RETR ' + filename, f.write)
print(" File transfer: done")
except Exception as e:
continue
for device in devices:
print("Connect to {}:".format(device['ip']))
#Создание сокета и объекта устройства
try:
s = libapi.socketOpen(device['ip'])
except Exception as e:
continue
dev_api = libapi.ApiRos(s)
import ftplib
import ftputil
import io
import gzip
from xml.dom.minidom import parse, parseString
import shutil
import sys
import socket
import os
# ftp 서버로부터 pubmed 데이터를 가져온다.
ftp = ftplib.FTP('ftp.ncbi.nlm.nih.gov')
ftp.login()
ftp.cwd('/pubmed/baseline')
file_mapper = {}
for file in ftp.nlst():
print(file)
# md5파일은 생략한다.
if len(file.split('.xml.gz.md5')) > 1: continue
# 압축 파일만 받아서 compressed data 파일에 쓴다.
if len(file.split('.xml.gz')) > 1:
f = open('./compressed_data/' + file, 'wb+').write
file_mapper[file] = ftp.retrbinary('RETR ' + file, f)
# 기존에 있는 파일을 가져다가 사용하는 경우
# files = os.listdir(path = "./compressed_data")
count = 0
# for key in files:
# xml 파일 압축 해제
for key in file_mapper:
fichier_index.write(" </tbody>\n")
fichier_index.write(" </table>\n")
fichier_index.write(
" <img id='check' src='../transparent.png' width='75'>\n")
fichier_index.write(
" <img id='check2' src='../transparent.png' width='75'>\n")
fichier_index.write(" </body>\n")
fichier_index.write("</html>\n")
fichier_index.close()
if transfert_ftp == '1':
print('\n\nFTP start')
repertoire_imagme_global = '/photos_numeriques/'
session = ftplib.FTP('www.imagme.com', 'godimagme', 'Kil01m@r')
for root, dirs, files in os.walk(dir_transfert, topdown=False):
print('------------------------------------------------------')
repertoire_raspberry = os.path.basename(root)
if not repertoire_raspberry == '':
try:
session.mkd(repertoire_imagme_global + repertoire_raspberry)
print('\nCreation repertoire: ' + '[www.imagme.com] ' +
repertoire_imagme_global + repertoire_raspberry)
except ftplib.error_perm:
pass
for f in files:
import ftplib
def getFile(ftp, filename):
try:
ftp.retrbinary("RETR " + filename, open(filename, 'wb').write)
except:
print("Error")
ftp = ftplib.FTP("ftp.nluug.nl")
ftp.login("anonymous", "ftplib-example-1")
ftp.cwd('/pub/') # change directory to /pub/
getFile(ftp, 'README.nluug')
ftp.quit()
def main():
''' connect to public ftp function '''
ftp = ftplib.FTP(url.netloc)
ftp.login()
print('login to ' + url.netloc)
logging.info('login to ' + url.netloc)
stack = [url.path]
path = stack.pop()
ftp.cwd(path)
# add all directories to the queue
children = ftp_dir(ftp)
dirs = [
posixpath.join(path, child[1]) for child in children if not child[0]
]
# set start_from directory
while True:
itemdir = dirs[0]
if itemdir.split('/')[-1] != start_from.strip():
del dirs[0]
else:
break
# put values from Illinois Entities.xlsx in dictionary
print('Creating connection with ' + illinois_entities_xlsx_file)
wbShort = openpyxl.load_workbook(dir_in +
illinois_entities_xlsx_file.strip())
sheetShort = wbShort.get_sheet_by_name(illinois_entities_sheet.strip())
dshort = {}
row = 2
scrolldown = True
while scrolldown:
key = str(sheetShort['A' + str(row)].value)
if len(key) == 6:
key = '00' + key
elif len(key) == 7:
key = '0' + key
dshort[key] = sheetShort['B' + str(row)].value.strip()
row += 1
if sheetShort['A' + str(row)].value == None:
scrolldown = False # when finding empty row parsing of Shortnames xlsx will stop
for udir in dirs:
print('-' * 20)
logging.info('-' * 20)
print(udir)
logging.info(udir)
# example of path structure /LocGovAudits/FY2015/00100000
parseddir = udir.split('/')[-1].strip()
try:
preparename = 'IL ' + dshort[parseddir] + ' ' + year + '.pdf'
except:
preparename = parseddir + '.pdf'
preparename = preparename.replace('/', '')
preparename = preparename.replace(':', '')
ftp.cwd(udir)
time.sleep(0.8)
files = []
try:
files = ftp.nlst()
files.sort()
except Exception as e:
if str(e) == "550 No files found":
print("No files in this directory")
logging.info(udir + " No files in this directory")
else:
print(str(e))
logging.info(udir + ' ' + str(e))
for f in files:
with open(dir_pdfs + f, 'wb') as fobj:
ftp.retrbinary('RETR %s' % f, fobj.write)
print('downloading ' + f)
logging.info('downloading ' + f)
# if more then one pdf in ftp directory merge them
if len(files) > 1:
pdfline = ' '.join(files)
if platform.system() == "Linux":
command = 'pdftk ' + pdfline + ' cat output temp.pdf'
if platform.system() == "Windows":
command = 'pdftk.exe ' + pdfline + ' cat output temp.pdf'
try:
os.system(command)
os.rename('temp.pdf', preparename)
print(preparename + ' generated')
logging.info(preparename + ' generated')
bOK = True
except Exception as e:
print(udir + ' ' + pdfline + ' not generated pdf')
print(str(e))
logging.info(udir + ' ' + pdfline + ' not generated pdf')
logging.info(str(e))
bOK = False
else:
# check is there only one pdf file
if len(files) == 1:
try:
os.rename(dir_pdfs + files[0].strip(),
dir_pdfs + preparename)
except Exception as e:
logging.info(str(e))
print(str(e))
print(preparename + ' generated')
logging.info(preparename + ' generated')
else:
print('no files in ' + udir)
logging.info('no files in ' +
udir) #this most probably will never occure
# delete original pdf files if more then one, since if one only, with renaming it is deleted
if len(files) > 1 and bOK:
for f in files:
os.remove(dir_pdfs + str(f).strip())
def _connect(self):
""" Log in to ftp """
self.ftpcon = ftplib.FTP(self.host, self.username, self.password)
#!/usr/bin/env python
import ftplib
ftp = ftplib.FTP()
ftp.set_debuglevel(2)
ftp.connect('192.168.124.128')
ftp.login()
print ftp.getwelcome()
ftp.cwd('nide')
ftp.dir()
bufsize = 1024
filename = 'nide'
file_handler = open(filename, 'wb').write
ftp.retrbinary('RETR nide', file_handler, bufsize)
ftp.set_debuglevel(0)
ftp.quit()
#!/usr/bin/python
import sys
import ftplib
import os
import time
server = "172.16.40.40"
user = "******"
password = "******"
source = "/Learning/Books/A+"
destination = "/home/technician/Downloads"
interval = 0.05
ftp = ftplib.FTP(server)
ftp.login(user, password)
def downloadFiles(path, destination):
try:
ftp.cwd(path)
os.chdir(destination)
mkdir_p(destination[0:len(destination) - 1] + path)
print "Created: " + destination[0:len(destination) - 1] + path
except OSError:
pass
except ftplib.error_perm:
print "Error: could not change to " + path
sys.exit("Ending Application")
filelist = ftp.nlst()
def run_program():
while running == True:
montemps=time.time()
#On récupère la liste des fichiers présent dans le dossier trié par date de modification
tab_txt = sorted(glob.glob(current_dir + str(extension)), key=os.path.getmtime)
nbficht = True
for elem in tab_txt:
tps1 = time.clock()
con = sqlite3.connect('ftdb.sql')
cur = con.cursor()
if nbficht == True:
logger.info(".........Fichiers trouvés: " + str(len(tab_txt)) + ".........")
logger.info('-----------------------------------------------------------')
nbficht = False
if os.path.isfile(elem) == True:
# On se connecte au serveur ftp
try:
ftp = ftplib.FTP(host)
ftp.login(user, password)
print "Connexion au serveur ftp réussie"
logger.info("Connexion au serveur ftp réussie")
logger.info('-----------------------------------------------------------')
except Exception, e:
print 'Erreur connexion ftp :' + str(e)
logger.error('Erreur connexion ftp :' + str(e))
logger.warning('-----------------------------------------------------------')
time.sleep(600)
#On envois le fichier 'elem' récupéré dans la boucle
try:
fic_open = open(str(elem), 'rb+')
ftp.storbinary('STOR ' + str(os.path.basename(elem)), fic_open)
fic_open.close()
success = True
print "Transfert de " + str(os.path.basename(elem)) + " effectué"
logger.info("Transfert de " + str(os.path.basename(elem)) + " effectué")
logger.info('-----------------------------------------------------------')
except Exception, e:
print 'Erreur lors du transfert du fichier: ' + str(e)
logger.error('Erreur lors du transfert du fichier: ' + str(e))
logger.warning('-----------------------------------------------------------')
success = False
#On vérifie si le fichier est bien présent sur le serveur distant uniquement si le transfert c'est bien passé
try:
if success == True:
ftp_dist = ftp.nlst() #Instruction permettant de lister le répertoire courant du ftp
if os.path.basename(elem) in ftp_dist:
ftp_success = True
elemtabmin = os.path.basename(elem).split('.')
elemOKmin = elemtabmin[0] + '.OK'
print elemOKmin
elemtabs = elem.split('.')
OKdir = str(elemtabs[0]) + '.OK'
print OKdir
elemOK = open(OKdir, "w")
elemOK.close()
time.sleep(0.1)
ftp.storbinary('STOR ' + elemOKmin, open(OKdir))
print "Le fichier " + str(os.path.basename(elem)) + " a bien été transféré sur le ftp distant."
logger.info("Le fichier " + str(os.path.basename(elem)) + " a bien été transféré sur le ftp distant.")
logger.info('-----------------------------------------------------------')
else:
ftp_success = False
else:
print "Impossible de tester si le fichier existe sur le serveur ftp"
logger.error("Impossible de tester si le fichier existe sur le serveur ftp")
logger.warning('-----------------------------------------------------------')
except Exception, e:
print "Impossible de récupérer la liste des fichiers distant: " + str(e)
logger.error("Impossible de récupérer la liste des fichiers distant: " + str(e))
logger.warning('-----------------------------------------------------------')
#On vérifie que le fichier a bien été envoyé, et si c'est le cas, sois on le backup, sois un le supprime du répertoire local
try:
if ftp_success == True:
maintenant = datetime.now()
a = maintenant.date()
b = maintenant.hour
cur.execute("SELECT nbfichier FROM donnees_transmises WHERE date = '" + str(a) + "' AND num_heure = " + str(b) )
all1 = cur.fetchall()
if all1 != []:
print all1[0][0]
all2 = int(all1[0][0]) + 1
print all2
print "liste pas vide donc update"
cur.execute("UPDATE donnees_transmises SET nbfichier =" + str(all2) + " WHERE date ='" + str(a) + "' AND num_heure =" + str(b))
con.commit()
else:
print "liste vide"
cur.execute("""INSERT INTO donnees_transmises (date, num_heure, nbfichier) VALUES(?,?,?)""", (a, b, "1"))
con.commit()
cur.close()
con.close()
except Exception, e:
print "Erreur lors du remplissage de la base de données sqlite3: " + str(e)
def upload(filename):
session = ftplib.FTP('192.168.2.29')
file = open(filename, "rb")
session.storbinary('STOR ' + filename, file)
file.close()
session.quit()
def attack(username,password,tgtHost,redirect): ftp = ftplib.FTP(tgtHost) ftp.login(username,password) defPages = returnDefault(ftp) for defPage in defPages: injectPage(ftp,defPage,redirect)
else:
#10-bit method
for line in data:
#append 0 to string for any '-', else 1
for letter in line:
if (letter == ('-')):
string += "0"
else:
string += "1"
#output the 7-bit ASCII version of the binary string
convertASCII(string, len(string))
#####################################MAIN#######################################
#setup connection to server
server = ftplib.FTP()
server.connect(SERVER)
server.login('anonymous')
#retrieve the permissions data from server
retrieveData()
#generate the binary string and print the 7-bit ASCII version
genString()
################################################################################
sys.exit(1)
print('success')
# get check-out file list from HOST:VIEW
print('\ncheck-out files in ' + view + ':')
co = doshellcmd(tn, 'lsco')[1:]
if not co:
print('no checkout files')
telnetlogout(tn)
sys.exit(1)
for file in co:
print(file)
# upload local files corresponding to check-outs to HOME@HOST
sys.stdout.write('\nftp ' + host + '...')
ftp = ftplib.FTP(host, user, password)
if not ftp:
print('fail')
telnetlogout(tn)
sys.exit(1)
print('success, upload files to ' + host + '/home/' + user)
for file in co:
fname = os.path.normpath(rootdir + file)
if not os.path.isfile(fname):
print(fname + ' not found')
continue
print(fname)
cmd = 'STOR ' + ':'.join(
file.split('/')) # rename file to avoid name conflict
ftp.storbinary(cmd, open(fname))
ftp.quit()
def connect():
"""Connect to FTP server, login and return an ftplib.FTP instance."""
ftp = ftplib.FTP()
ftp.connect(HOST, PORT)
ftp.login(USER, PASSWORD)
return ftp
def get_url(self,
url,
dest,
makedirs=False,
saltenv='base',
no_cache=False,
cachedir=None):
'''
Get a single file from a URL.
'''
url_data = urlparse(url)
url_scheme = url_data.scheme
url_path = os.path.join(url_data.netloc, url_data.path).rstrip(os.sep)
# If dest is a directory, rewrite dest with filename
if dest is not None \
and (os.path.isdir(dest) or dest.endswith(('/', '\\'))):
if url_data.query or len(
url_data.path) > 1 and not url_data.path.endswith('/'):
strpath = url.split('/')[-1]
else:
strpath = 'index.html'
if salt.utils.is_windows():
strpath = salt.utils.sanitize_win_path_string(strpath)
dest = os.path.join(dest, strpath)
if url_scheme and url_scheme.lower() in string.ascii_lowercase:
url_path = ':'.join((url_scheme, url_path))
url_scheme = 'file'
if url_scheme in ('file', ''):
# Local filesystem
if not os.path.isabs(url_path):
raise CommandExecutionError(
'Path \'{0}\' is not absolute'.format(url_path))
if dest is None:
with salt.utils.fopen(url_path, 'r') as fp_:
data = fp_.read()
return data
return url_path
if url_scheme == 'salt':
result = self.get_file(url,
dest,
makedirs,
saltenv,
cachedir=cachedir)
if result and dest is None:
with salt.utils.fopen(result, 'r') as fp_:
data = fp_.read()
return data
return result
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return ''
elif not no_cache:
dest = self._extrn_path(url, saltenv, cachedir=cachedir)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
def s3_opt(key, default=None):
'''Get value of s3.<key> from Minion config or from Pillar'''
if 's3.' + key in self.opts:
return self.opts['s3.' + key]
try:
return self.opts['pillar']['s3'][key]
except (KeyError, TypeError):
return default
self.utils['s3.query'](method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=s3_opt('key'),
keyid=s3_opt('keyid'),
service_url=s3_opt('service_url'),
verify_ssl=s3_opt('verify_ssl', True),
location=s3_opt('location'),
path_style=s3_opt('path_style', False),
https_enable=s3_opt(
'https_enable', True))
return dest
except Exception as exc:
raise MinionError(
'Could not fetch from {0}. Exception: {1}'.format(
url, exc))
if url_data.scheme == 'ftp':
try:
ftp = ftplib.FTP()
ftp.connect(url_data.hostname, url_data.port)
ftp.login(url_data.username, url_data.password)
with salt.utils.fopen(dest, 'wb') as fp_:
ftp.retrbinary('RETR {0}'.format(url_data.path), fp_.write)
ftp.quit()
return dest
except Exception as exc:
raise MinionError(
'Could not retrieve {0} from FTP server. Exception: {1}'.
format(url, exc))
if url_data.scheme == 'swift':
try:
def swift_opt(key, default):
'''Get value of <key> from Minion config or from Pillar'''
if key in self.opts:
return self.opts[key]
try:
return self.opts['pillar'][key]
except (KeyError, TypeError):
return default
swift_conn = SaltSwift(swift_opt('keystone.user', None),
swift_opt('keystone.tenant', None),
swift_opt('keystone.auth_url', None),
swift_opt('keystone.password', None))
swift_conn.get_object(url_data.netloc, url_data.path[1:], dest)
return dest
except Exception:
raise MinionError('Could not fetch from {0}'.format(url))
get_kwargs = {}
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
netloc = url_data.netloc
at_sign_pos = netloc.rfind('@')
if at_sign_pos != -1:
netloc = netloc[at_sign_pos + 1:]
fixed_url = urlunparse(
(url_data.scheme, netloc, url_data.path, url_data.params,
url_data.query, url_data.fragment))
get_kwargs['auth'] = (url_data.username, url_data.password)
else:
fixed_url = url
destfp = None
try:
# Tornado calls streaming_callback on redirect response bodies.
# But we need streaming to support fetching large files (> RAM
# avail). Here we are working around this by disabling recording
# the body for redirections. The issue is fixed in Tornado 4.3.0
# so on_header callback could be removed when we'll deprecate
# Tornado<4.3.0. See #27093 and #30431 for details.
# Use list here to make it writable inside the on_header callback.
# Simple bool doesn't work here: on_header creates a new local
# variable instead. This could be avoided in Py3 with 'nonlocal'
# statement. There is no Py2 alternative for this.
#
# write_body[0] is used by the on_chunk callback to tell it whether
# or not we need to write the body of the request to disk. For
# 30x redirects we set this to False because we don't want to
# write the contents to disk, as we will need to wait until we
# get to the redirected URL.
#
# write_body[1] will contain a tornado.httputil.HTTPHeaders
# instance that we will use to parse each header line. We
# initialize this to False, and after we parse the status line we
# will replace it with the HTTPHeaders instance. If/when we have
# found the encoding used in the request, we set this value to
# False to signify that we are done parsing.
#
# write_body[2] is where the encoding will be stored
write_body = [None, False, None]
def on_header(hdr):
if write_body[1] is not False and write_body[2] is None:
# Try to find out what content type encoding is used if
# this is a text file
write_body[1].parse_line(hdr) # pylint: disable=no-member
if 'Content-Type' in write_body[1]:
content_type = write_body[1].get('Content-Type') # pylint: disable=no-member
if not content_type.startswith('text'):
write_body[1] = write_body[2] = False
else:
encoding = 'utf-8'
fields = content_type.split(';')
for field in fields:
if 'encoding' in field:
encoding = field.split('encoding=')[-1]
write_body[2] = encoding
# We have found our encoding. Stop processing headers.
write_body[1] = False
# If write_body[0] is False, this means that this
# header is a 30x redirect, so we need to reset
# write_body[0] to None so that we parse the HTTP
# status code from the redirect target.
if write_body[0] is write_body[1] is False:
write_body[0] = None
# Check the status line of the HTTP request
if write_body[0] is None:
try:
hdr = parse_response_start_line(hdr)
except HTTPInputError:
# Not the first line, do nothing
return
write_body[0] = hdr.code not in [301, 302, 303, 307]
write_body[1] = HTTPHeaders()
if no_cache:
result = []
def on_chunk(chunk):
if write_body[0]:
if write_body[2]:
chunk = chunk.decode(write_body[2])
result.append(chunk)
else:
dest_tmp = "{0}.part".format(dest)
# We need an open filehandle to use in the on_chunk callback,
# that's why we're not using a with clause here.
destfp = salt.utils.fopen(dest_tmp, 'wb') # pylint: disable=resource-leakage
def on_chunk(chunk):
if write_body[0]:
destfp.write(chunk)
query = salt.utils.http.query(fixed_url,
stream=True,
streaming_callback=on_chunk,
header_callback=on_header,
username=url_data.username,
password=url_data.password,
opts=self.opts,
**get_kwargs)
if 'handle' not in query:
raise MinionError('Error: {0} reading {1}'.format(
query['error'], url))
if no_cache:
if write_body[2]:
return six.u('').join(result)
return six.b('').join(result)
else:
destfp.close()
destfp = None
salt.utils.files.rename(dest_tmp, dest)
return dest
except HTTPError as exc:
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
exc.code, url,
*BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
except URLError as exc:
raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
finally:
if destfp is not None:
destfp.close()
def ftp_file_upload(connection_parameter, file_list):
##############################################
# Function to chek valid input format
# Arguments:
# a dictionary with all the connection parameters
# a list containing a list() of file path to upload
#
# connection_parameter = {'srv_address': 'xxx.xxx.xxx.xxx',
# 'ftp_usr': '******',
# 'ftp_psw': 'password',
# 'ftp_path': '/path/to/upload/'}
# Returns:
# true if all files were uploaded
# false if something goes wrong
##############################################
# TODO: Handle upload of different file type based on extension.
logger = logging.getLogger(__name__)
# Server address is a requirement.
if not connection_parameter['srv_address']:
logger.error("No FTP server address")
return False
if not file_list:
logger.error("No data to transmit")
return False
remote = ftplib.FTP()
try:
logger.info("Connecting to %s", connection_parameter['srv_address'])
remote.connect(connection_parameter['srv_address'], 21, 60)
remote.login(connection_parameter['ftp_usr'],
connection_parameter['ftp_psw'])
# Forcing passive mode
remote.set_pasv(True)
logger.info("Moving into remote dir: %s",
connection_parameter['ftp_path'])
remote.cwd(connection_parameter['ftp_path'])
except ftplib.all_errors as ftperr:
logger.error("Error during FTP transmission: %s",
ftperr,
exc_info=True)
return False
if not file_list:
logger.warning("No file passed for upload")
remote.close()
return False
# List for upload status
upload_status = list()
for upload in file_list:
if upload == '':
logger.error("Cowardly refusing to transmit an empty string...")
upload_status.append(False)
continue
logger.info("Uploading file: %s", upload)
try:
with open(upload, 'rb') as fp:
# Getting only the filename for
# ftp server compatibility (filezilla)
filename = upload.split("/")[-1]
# remote.storlines('STOR ' + filename, fp)
remote.storbinary('STOR ' + filename, fp)
upload_status.append(True)
except IOError as file_err:
logger.error("Error transferring file %s", upload)
logger.error("Error during FILE operation: %s",
file_err,
exc_info=True)
upload_status.append(False)
except ftplib.all_errors as ftp_err:
logger.error("Error transferring file %s", upload)
logger.error("Error during FILE operation: %s",
ftp_err,
exc_info=True)
upload_status.append(False)
logger.info("Closing connection to %s",
connection_parameter['srv_address'])
remote.close()
if False in upload_status:
return False
# If everything is fine exit with true!
return True
"ftp://ftp.sra.ebi.ac.uk/vol1/fastq/SRR507/008/SRR5073518/SRR5073518_1.fastq.gz",
"ftp://ftp.sra.ebi.ac.uk/vol1/fastq/SRR507/008/SRR5073518/SRR5073518_2.fastq.gz",
"ftp://ftp.sra.ebi.ac.uk/vol1/fastq/SRR507/005/SRR5073705/SRR5073705_1.fastq.gz",
"ftp://ftp.sra.ebi.ac.uk/vol1/fastq/SRR507/005/SRR5073705/SRR5073705_2.fastq.gz",
"ftp://ftp.sra.ebi.ac.uk/vol1/fastq/SRR507/007/SRR5073987/SRR5073987_1.fastq.gz",
"ftp://ftp.sra.ebi.ac.uk/vol1/fastq/SRR507/007/SRR5073987/SRR5073987_2.fastq.gz"
]
def getFile(ftp, filename):
try:
ftp.retrbinary("RETR " + filename, open(filename, 'wb').write)
except:
print("Error")
ftp = ftplib.FTP("ftp.sra.ebi.ac.uk")
ftp.login()
for link in links_of_genome_r_files:
file_location = "/" + "/".join(link.split("/")[3:-1]) + "/"
file_name = link.split("/")[-1]
print("\n##############\n")
print("File Name => ", file_name)
#print("File Location => ", file_location)
ftp.cwd(file_location)
getFile(ftp, file_name)
print("\n##############\n")
ftp.quit()
unit='bit')
def update_download(data):
file.write(data)
progress_bar.update(len(data))
ftp.retrbinary('RETR ' + remote_file, update_download)
print('\n%s: Download finished\n' % time.ctime())
except ftplib.error_perm:
return
ip = 'ftp.merrbys.co.uk'
login = '******'
password = '******'
with ftplib.FTP(ip, login, password) as ftp:
search_list = []
with open('kmz_search_output.txt') as kmz_search_file:
for line in kmz_search_file:
if 'File' in line:
search_list.append(line)
for count, line in enumerate(search_list):
print('\nFile %s of %s\n' % (count, len(search_list)))
file_string = line.split(' ')[1].strip()
ddm_remote_file = os.path.join(
'/Data/L1B/',
os.path.basename(file_string).replace('.', '/').replace(
'kmz', 'DDMs.nc'))
metadata_remote_file = os.path.join(
'/Data/L1B/',
"""
import os
import sys
import ftplib
from getpass import getpass
from mimetypes import guess_type
nonpassive = False # 默认为被动FTP
remotesite = 'home.rmi.net' # 从这个站点下载
remotedir = '.' # 这个目录(如public_html)
remoteuser = '******'
remotepass = getpass('Password for %s on %s: ' % (remoteuser, remotesite))
localdir = (len(sys.argv) > 3 and sys.argv[1]) or '.'
cleanall = input('Clean local directory first? ')[:1] in ['y', 'Y']
print('connecting...')
connection = ftplib.FTP(remotesite) # 连接到FTP站点
connection.login(remoteuser, remotepass) # 用户名/密码登录
connection.cwd(remotedir) # 进入待复制目录
if nonpassive: # 强制主动FTP模式
connection.set_pasv(False) # 多数服务器采用被动模式
if cleanall:
for localname in os.listdir(localdir): # 尝试删除所有本地文本
try:
print('deleting local', localname)
os.remove(os.path.join(localdir, localname))
except:
print('cannot delete local', localname)
count = 0 # 下载所有远程文件
remotefiles = connection.nlst() # nlst()给出文件列表
for remotename in remotefiles:
ap = argparse.ArgumentParser(usage=usage)
ap.add_argument('--dest', default=".", help="Destination directory; "
"FlyBase GFF will be downloaded here and database will be created")
ap.add_argument('--list', action="store_true", help="Just list the available"
"chromosomes to download")
ap.add_argument('--chrom', default='all-no-analysis', help="chromosome"
" or version to download, one of %s" % CHROMS)
ap.add_argument('--make_db', action='store_true',
help="Create a gffutils database from the downloaded file")
args = ap.parse_args()
URL = "ftp.flybase.net"
PATH = "genomes/dmel/current/gff/"
ftp = ftplib.FTP(URL)
sys.stderr.write(ftp.login())
dirlist = ftp.nlst(PATH)
sys.stderr.write('\n')
for i in dirlist:
size = ftp.size(i)
if ('-' + args.chrom + '-') in i:
fn = i
flag = 'X'
else:
flag = " "
sys.stderr.write("[ %s ] [%.1f MB] %s\n" % (flag, (size / 1e6), i))
size = ftp.size(fn)
def on_message(client, userdata, msg):
global timestamp_previous_end, scheduler
# Split the mqtt message into a word list.
mqtt_msg = msg.payload.decode()
mqtt_msg_wordlist = mqtt_msg.split()
print(mqtt_msg_wordlist)
if mqtt_msg_wordlist[0] == MQTT_POST_EXPS:
batch_file = mqtt_msg_wordlist[1]
username = mqtt_msg_wordlist[2]
# Get the batch file from ftp
with ftplib.FTP() as ftp:
ftp.set_pasv(FTP_PASSIVE_MODE)
ftp.connect(FTP_ADDRESS, FTP_PORT)
print(ftp.getwelcome())
try:
ftp.login(FTP_USER, FTP_PASSWORD)
ftp.cwd('/home/' + FTP_USER + '/' + username + '/')
print(batch_file)
file_size = ftp.size(batch_file)
if(file_size > 0):
grab_file_from_ftp(batch_file, TESTBED_TMP_PATH + '/batch_file.txt', ftp)
f = open(TESTBED_TMP_PATH + '/batch_file.txt', "r")
batch_list = f.readlines()
f.close()
if(len(exp_time_list) == 0):
date_time_now = datetime.datetime.now()
time_now = date_time_now.time()
timestamp_now = date_time_now.replace(tzinfo = None).timestamp()
if(timestamp_now > (timestamp_previous_end + experiment_gap)):
timestamp_previous_end = timestamp_now
print(timestamp_previous_end)
for tasks in batch_list:
tasks_wl = tasks.split()
experiment_duration = int(tasks_wl[2])
print(tasks_wl)
# Compute time plus the delta of preparation.
timestamp_actural_start = timestamp_previous_end + preparation_time
experiment_start_time = datetime.datetime.fromtimestamp(timestamp_actural_start)
experiment_start_time = experiment_start_time.replace(microsecond = 0)
# Compute the experiment ID and schedule it.
exp_id = tid_maker(username)
print(exp_id)
scheduler.add_job(testbed_scheduler_function, 'date', run_date = experiment_start_time, args = [[tasks, exp_id]], id = exp_id)
print(experiment_start_time)
# Compute end time of the experiment
experiment_start_time_timestamp = experiment_start_time.replace(tzinfo = None).timestamp()
experiment_end_time_timestamp = experiment_start_time_timestamp + experiment_duration * 60
timestamp_previous_end = experiment_end_time_timestamp + experiment_gap
experiment_end_time = datetime.datetime.fromtimestamp(experiment_end_time_timestamp)
experiment_end_time = experiment_end_time.replace(microsecond = 0)
# Generate and new element in the exp_time_list and append it.
exp_t = exp_time()
exp_t.name = tasks_wl[1]
exp_t.experiment_id = exp_id
exp_t.start = experiment_start_time_timestamp
exp_t.end = experiment_end_time_timestamp
exp_time_list.append(exp_t)
print(exp_t)
print(scheduler.get_jobs())
except ftplib.all_errors as e:
print('FTP error: ', e)
def __genFileIndex(self, version, basedir, release, organism, where):
files = {}
finalPath = '%s/genomes/%s' % (
(settings.getDataDir() if where is None else where), version)
conn = ftplib.FTP("ftp.ensembl.org")
conn.login()
organisms = [
line.split('/')[-1]
for line in conn.nlst("%s/release-%d/gff3" % (basedir, release))
]
if organism not in organisms:
utils.msg.warning("Organism '%s' not in release %d" %
(organism, release))
return {}
#fi
def genGFF3():
# This is ugly, but the most stable way I was able to find the correct GFF3 file.
# in GRCH37, it didnt match the obvious pattern...
uri = [
u[0] for u in sorted(
[(line, len(line.split('.')))
for line in conn.nlst("%s/release-%d/gff3/%s" %
(basedir, release, organism))
if (len(line.split('.')) > 3)],
key=lambda x: x[1])
]
#uri = [ line for line in conn.nlst("%s/release-%d/gff3/%s" % (basedir, release, organism)) if '%d.gff3.gz' % release in line ]
if len(uri) > 0:
return utils.Acquire(where=where).curl(
"ftp://ftp.ensembl.org/%s" % uri[0]).gunzip().finalize(
'%s/genes.gff3' % finalPath)
#fi
return None
#edef
def genGenome():
uri = [
line for line in conn.nlst("%s/release-%d/fasta/%s/dna" %
(basedir, release, organism))
if 'dna.chromosome' in line
]
aos = [
utils.Acquire(where=where).curl("ftp://ftp.ensembl.org/%s" % f)
for f in uri
]
return utils.Acquire(where=where).merge(
aos, method='zcat').finalize('%s/dna.fasta' % finalPath)
#edef
def genCDS():
uri = [
line for line in conn.nlst("%s/release-%d/fasta/%s/cds" %
(basedir, release, organism))
if 'fa.gz' in line
]
if len(uri) > 0:
return utils.Acquire(where=where).curl(
"ftp://ftp.ensembl.org/%s" % uri[0]).gunzip().finalize(
'%s/cds.fa' % finalPath)
#fi
return None
#edef
def genAA():
uri = [
line for line in conn.nlst("%s/release-%d/fasta/%s/pep" %
(basedir, release, organism))
if 'all.fa.gz' in line
]
if len(uri) > 0:
return utils.Acquire(where=where).curl(
"ftp://ftp.ensembl.org/%s" % uri[0]).gunzip().finalize(
'%s/aa.fa' % finalPath)
#fi
return None
#edef
def genIDS():
def idMapFunc(inFile, outFile):
fasta = formats.Fasta(inFile)
with open(outFile, 'w') as ofd:
ofd.write('\t'.join(
['gene', 'transcript', 'protein', 'symbol']) + '\n')
for seq in fasta:
fullName = fasta[seq].fullName
data = dict([(p.split(':')[0],
':'.join(p.split(':')[1:]))
for p in fullName.split(' ')
if p.split(':')[0] in [
'chromosome', 'gene', 'transcript',
'gene_symbol'
]])
ofd.write('\t'.join([
data.get('gene', ''),
data.get('transcript', ''), seq,
data.get('gene_symbol', '')
]))
ofd.write('\n')
#efor
#ewith
return 0
#edef
uri = [
line for line in conn.nlst("%s/release-%d/fasta/%s/pep" %
(basedir, release, organism))
if 'all.fa.gz' in line
]
if len(uri) > 0:
return utils.Acquire(where=where).curl(
"ftp://ftp.ensembl.org/%s" %
uri[0]).gunzip().func(idMapFunc).finalize('%s/ids.tsv' %
finalPath)
#fi
return None
#edef
files["gff"] = genGFF3()
files["genome"] = genGenome()
files["cds"] = genCDS()
files["aa"] = genAA()
files["ids"] = genIDS()
for f in files:
if f is None:
del files[f]
#fi
#efor
conn.close()
return files
######################################################################
# Networking
# ----------
#
# TODO
######################################################################
# FTP
# ~~~
#
# Full FTP features with ftplib
import ftplib
ftp = ftplib.FTP("ftp.cea.fr")
ftp.login()
ftp.cwd('/pub/unati/people/educhesnay/pystatml')
ftp.retrlines('LIST')
fd = open(os.path.join(tmpdir, "README.md"), "wb")
ftp.retrbinary('RETR README.md', fd.write)
fd.close()
ftp.quit()
# File download urllib
import urllib.request
ftp_url = 'ftp://ftp.cea.fr/pub/unati/people/educhesnay/pystatml/README.md'
urllib.request.urlretrieve(ftp_url, os.path.join(tmpdir, "README2.md"))
######################################################################
import ftplib
import os
import socket
HOST = 'ftp.acc.umu.se'
DIR = 'Pubic/EFLIB/'
FILE = 'README'
# 1. 客户端链接远程主机上的FTP服务器
try:
f = ftplib.FTP()
# 通过设置调试级可以方便调试
f.set_debuglevel(2)
# 链接主机地址
f.connect(HOST)
except Exception as e:
print(e)
exit()
print('Connected to host {0}'.format(HOST))
# 2. 客户端输入用户名和密码（或者'anonymous'和电子邮件地址）
try:
# 登录如果没有输入用户信息，则默认使用匿名登录
f.login()
except Exception as e:
print(e)
exit()
print('Logged in as "anonymous"')
# 3. 客户端和服务器进行各种文件传输和信息查询操作
years = numpy.arange(2007, 2014, 1).tolist()
url = 'ftp://ftp.meteo.uni-bonn.de/pub/reana/COSMO-REA2/TOT_PREC/'
output_folder = '/Volumes/Untitled/COSMO/REA2'
"""
file = r'/Users/janliechti/Google Drive/UNI/FS19/Geographie/Gedatenanalyse_u_Modellierung/Project_COSMO/PyCharm_project/Data/TOTAL_PRECIPITATION.SFC.200701.grb'
file_netcdf = r'/Users/janliechti/Google Drive/UNI/FS19/Geographie/Gedatenanalyse_u_Modellierung/Project_COSMO/PyCharm_project/Data/test.nc'
path_export_folder = r'/Users/janliechti/Google Drive/UNI/FS19/Geographie/Gedatenanalyse_u_Modellierung/Project_COSMO/PyCharm_project'
ds = xr.open_dataset(file, engine='cfgrib')
"""
# COSMO Rea2
for y in years:
# download files directly via python
ftp = ftplib.FTP(r'ftp.meteo.uni-bonn.de')
ftp.login()
ftp.cwd('/pub/reana/COSMO-REA2/TOT_PREC/' + str(y))
grb_list = ftp.nlst() # List files in folder
os.makedirs(os.path.join(output_folder, str(y) + '_grib'))
os.makedirs(os.path.join(output_folder, str(y) + '_netcdf'))
for f in grb_list:
wget.download(url + str(y) + '/' + f, os.path.join(output_folder, str(y) + '_grib', f))
ds = xr.open_dataset(os.path.join(output_folder, str(y) + '_grib', f), engine='cfgrib')
time = ds.__getitem__('time') # Timesteps --> for each hour a raster
tp = ds.__getitem__('tp')
steps = ds.__getitem__('step')
