def sync(self):
"""
downloads all needed_files from self.hostname (FTP)
of the downloaded files, extracts .gz files to same local_working_dir
-using self.extract function
parses the .txt downloaded needed_files
-using the self.parse function
"""
ftps = FTP_TLS(self.hostname) # connect to host, default port
ftps.login(self.username, self.password)
ftps.prot_p()
ftps.cwd(self.remote_dir) # change into "logs" directory
ftps.retrlines('LIST *.gz *.txt', self.ftp_list_callback) # list directory contents
for needed_file in self.needed_files:
if self.logging:
print "Writing {0} to {1}...".format(needed_file, self.local_working_dir)
ftps.retrbinary("RETR " + needed_file, open(os.path.join(self.local_working_dir, needed_file), 'wb').write)
if self.logging:
print "done syncing files"
for needed_file in self.needed_files:
if needed_file.endswith(".gz"):
self.extract(os.path.join(self.local_working_dir, needed_file))
txt_file_name = needed_file.replace('.gz','')#if already a .txt file, this is unnceccessary but works.
self.parse(txt_file_name)
if self.logging:
print "done extracting/parsing .gz files"
ftps.quit()
def downloadSarZIP(value, ftpParams):
ftps = FTP_TLS()
#ftps.set_debuglevel(2)
ftps.connect(ftpParams.host)
ftps.sendcmd('USER ' + ftpParams.user)
ftps.sendcmd('PASS ' + ftpParams.password)
list_of_files = []
checkFile = []
ftps.retrlines("NLST", list_of_files.append)
# ...or use the existing helper
# list_of_files = ftps.nlst()
dest_dir = "./"
# download files from ftp
for name in list_of_files:
if fnmatch.fnmatch(name, value + "_*"):
checkFile.append(name)
with open(os.path.join(dest_dir, name), "wb") as f:
ftps.retrbinary("RETR {}".format(name), f.write)
#delete files from ftp
for name in list_of_files:
if fnmatch.fnmatch(name, value + "_*"):
ftps.delete(name)
ftps.quit()
return checkFile
def f_sending_order():
global l_selected
l = l_selected + [ent_cast.get(), txt_article.get(1.0, END), f_time_now()]
with open('client_app_order.pkl', 'wb') as f:
pickle.dump(l, f)
print('start sending......')
try:
ftps = FTP_TLS(helpic.place_ftp, helpic.user_ftp, helpic.password_ftp)
ftps.cwd('/www/{}'.format(helpic.directory_ftp))
except:
showinfo(
'Connection', 'You don\'t have internet conection\n\
or login/pasword were changed')
return None
sufix = random.randint(1, 1000000)
file_name = 'order{}.pkl'.format(sufix)
ftps.storbinary('STOR ' + file_name,
open('client_app_order.pkl',
'rb')) # загрузка файла НА сервер
print('finish sending')
ftps.quit()
f_get_items_table(
) # getting information from FTP-server and writing in file
def download_all_in_one_path(targetdir,resultdir,check = True,num = 50):
if(os.path.exists(resultdir) == False):
os.makedirs(resultdir)
ftp = FTP('129.164.179.23')
ftp.login()
ftp.prot_p()
ftp.cwd(targetdir)
files = ftp.nlst()
target = 'https://heasarc.gsfc.nasa.gov/FTP' + targetdir
c = None
if(check):
c = []
data1 = []
ftp.voidcmd('TYPE I')
print('正在获取校验信息........')
for i in files:
#print(i)
data = os.path.join(target,i)
print(data)
data1.append(data)
if(check):
c.append(ftp.size(i))
ftp.quit()
if(check == False):
print('忽略数据大小校验。')
print('正在校验...............')
down(data1,resultdir,check=c,threadnum = num)
print('\n任务下载完成！！！')
def DeleteTextFile():
filename = 'test.txt'
session = FTP_TLS('files.000webhost.com', 'vulcuns-webserver',
'AKPJJDT2lol')
session.cwd('public_html')
session.delete(filename)
session.quit()
def send_chunks(self):
if self.final_chunks is None:
return ERR
if self.tls_flag:
if self.auth_flag:
ftp_obj = FTP_TLS(host=self.server,
user=self.creds[0],
passwd=self.creds[1])
else:
ftp_obj = FTP_TLS(host=self.server)
else:
if self.auth_flag:
ftp_obj = FTP(host=self.server,
user=self.creds[0],
passwd=self.creds[1])
else:
ftp_obj = FTP(host=self.server)
try:
ftp_obj.login()
sys.stdout.write("\t[+]\tConnected to server %s.\n" % self.server)
except:
sys.stderr.write("\t[-]\tCould not login to the server.\n")
return ERR
for chunk in self.final_chunks:
ftp_obj.mkd(chunk)
time.sleep(SLEEP)
ftp_obj.quit()
sys.stdout.write("\t[+]\tWrote %s(+1) folders.\n" %
(len(self.final_chunks) - 1))
return OKAY
def sendUpdateFilesFtp():
print "---- Send update files by FTP"
global serverFtp
global userFtp
global passFtp
from ftplib import FTP_TLS
ftps = FTP_TLS(serverFtp)
ftps.set_debuglevel(1)
ftps.login(userFtp, passFtp)
ftps.prot_p()
ftps.cwd('/files/updates')
ftps.storbinary('STOR ' + 'file_list.md5', open(quiterssFileRepoPath + '\\file_list.md5', 'rb'))
ftps.storbinary('STOR ' + 'VersionNo.h', open(quiterssSourcePath + '\\src\\VersionNo.h', 'rb'))
ftps.storbinary('STOR ' + 'HISTORY_EN', open(quiterssSourcePath + '\\HISTORY_EN', 'rb'))
ftps.storbinary('STOR ' + 'HISTORY_RU', open(quiterssSourcePath + '\\HISTORY_RU', 'rb'))
prepareFileList7z = []
for file in prepareFileList:
prepareFileList7z.append(file + '.7z')
for fileName in prepareFileList7z:
ftps.storbinary('STOR ' + 'windows' + fileName.replace('\\','/'), open(quiterssFileRepoPath + '\\windows' + fileName, 'rb'))
ftps.quit()
def getfilelist(server, port, user, password, db):
sqliteconnection = sqlite3.connect(db)
sqlitecursor = sqliteconnection.cursor()
sqlitecursor.execute('''CREATE TABLE IF NOT EXISTS files (date int, name text, CONSTRAINT 'id_UNIQUE' UNIQUE ('name'))''')
sqliteconnection.commit()
ftpsconnection = FTP_TLS()
ftpsconnection.connect(server, port)
ftpsconnection.auth()
ftpsconnection.prot_p()
ftpsconnection.login(user, password)
ftpsconnection.prot_p()
rootfiles = ftpsconnection.nlst()
for i in range(0,5):
episodes = ftpsconnection.nlst(rootfiles[i])
for episode in episodes:
sqlitecursor.execute('''INSERT OR IGNORE INTO files VALUES ("%(date)d", "%(folder)s")''' % {'date': time.time(), 'folder': ("/" + rootfiles[i] + "/" + episode) } )
sqliteconnection.commit()
sqliteconnection.close()
ftpsconnection.quit()
ftpsconnection.close()
def send_chunks(self):
if self.final_chunks is None:
return ERR
if self.tls_flag:
if self.auth_flag:
ftp_obj = FTP_TLS(host=self.server, user=self.creds[0], passwd=self.creds[1])
else:
ftp_obj = FTP_TLS(host=self.server)
else:
if self.auth_flag:
ftp_obj = FTP(host=self.server, user=self.creds[0], passwd=self.creds[1])
else:
ftp_obj = FTP(host=self.server)
try:
ftp_obj.login()
sys.stdout.write("\t[+]\tConnected to server %s.\n" % self.server)
except:
sys.stderr.write("\t[-]\tCould not login to the server.\n")
return ERR
for chunk in self.final_chunks:
ftp_obj.mkd(chunk)
time.sleep(SLEEP)
ftp_obj.quit()
sys.stdout.write("\t[+]\tWrote %s(+1) folders.\n" % (len(self.final_chunks)-1))
return OKAY
def Push( FtpServer, Username, Password, uploadlist = FilesToPut, port = 21, passive = False, StartTls = False ):
print "Login to %s:%s using %s:%s (%s%s)"%(FtpServer, port, Username, 'xxx',
'passive' if passive else 'active',
'/tls' if StartTls else '')
if StartTls:
ftp = FTP_TLS()
else:
ftp = FTP()
#ftp.set_debuglevel(2)
ftp.connect( FtpServer, port )
ftp.login( Username, Password ) # user anonymous, passwd anonymous@
ftp.set_pasv( passive )
if StartTls:
ftp.prot_p()
for f in uploadlist:
print "uploading %s"%f
fp = open( f, 'rb')
ftp.storbinary('STOR %s'%os.path.basename(f), fp) # send the file
ftp.quit()
def sendPackagesFtp():
print "---- Send packages by FTP"
global serverFtp
global userFtp
global passFtp
from ftplib import FTP_TLS
ftps = FTP_TLS(serverFtp)
ftps.set_debuglevel(1)
ftps.login(userFtp, passFtp)
ftps.prot_p()
try:
ftps.sendcmd('MKD ' + '/files/' + strProductVer)
except Exception:
print 'Directory already exists'
ftps.cwd('/files/' + strProductVer)
filesListFtp = ftps.nlst()
filesList = os.listdir(packagesPath)
newFilesList = [e for e in filesList if not(e in filesListFtp)]
for fileName in newFilesList:
ftps.storbinary('STOR ' + fileName, open(packagesPath + '\\' + fileName, 'rb'))
ftps.quit()
def doRequest():
# Send the file up to the FTP server
print("Upload picture start")
ftp = FTP_TLS('individual.utoronto.ca', 'USER', '1PASS.')
file = open('data.jpg', 'rb') # file to send
ftp.storbinary('STOR data.jpg', file) # send the file
file.close() # close file and FTP
ftp.quit()
print("Upload picture complete")
print("Requesting file request")
reqUrlA = 'http://api.cloudsightapi.com/image_requests/' # get token
reqUrlB = 'http://api.cloudsightapi.com/image_responses/' # get the final recognition result with token
headers = {
'Authorization': 'CloudSight 149xzcR0nYPrwThNXVLecQ',
}
postData = {
'image_request[remote_image_url]':
"http://individual.utoronto.ca/timlock/data.jpg",
'image_request[locale]': "en_us",
'image_request[language]': "en"
}
try:
response = requests.post(reqUrlA, headers=headers, params=postData)
except Exception, e:
print 'Error: connection error, please check your Internet and confirm the image url'
print e
return ("FAILED")
def doRequest():
# Send the file up to the FTP server
print ("Upload picture start")
ftp = FTP_TLS('individual.utoronto.ca', 'USER','1PASS.')
file = open('data.jpg','rb') # file to send
ftp.storbinary('STOR data.jpg', file) # send the file
file.close() # close file and FTP
ftp.quit()
print ("Upload picture complete")
print ("Requesting file request")
reqUrlA = 'http://api.cloudsightapi.com/image_requests/' # get token
reqUrlB = 'http://api.cloudsightapi.com/image_responses/' # get the final recognition result with token
headers = {
'Authorization' : 'CloudSight 149xzcR0nYPrwThNXVLecQ',
}
postData = {
'image_request[remote_image_url]' : "http://individual.utoronto.ca/timlock/data.jpg",
'image_request[locale]': "en_us",
'image_request[language]': "en"
}
try:
response = requests.post(reqUrlA, headers=headers, params=postData)
except Exception, e:
print 'Error: connection error, please check your Internet and confirm the image url'
print e
return ("FAILED")
def get_ftp_data(self, cr, uid, ids, context={}):
for chain in self.browse(cr, uid, ids, context=context):
config_obj = chain.ftp_config_id
try:
conn = FTP_TLS(host=config_obj.host,
user=config_obj.username,
passwd=config_obj.passwd)
conn.prot_p()
except:
conn = FTP(host=config_obj.host,
user=config_obj.username,
passwd=config_obj.passwd)
filenames = conn.nlst()
for filename in filenames:
input_file = StringIO()
conn.retrbinary('RETR %s' % filename,
lambda data: input_file.write(data))
input_string = input_file.getvalue()
input_file.close()
csv_reader = unicode_csv_reader(
StringIO(input_string),
delimiter=str(chain.separator),
quoting=(not chain.delimiter and csv.QUOTE_NONE)
or csv.QUOTE_MINIMAL,
quotechar=chain.delimiter and str(chain.delimiter) or None,
charset=chain.charset)
self.import_to_db(cr,
uid,
ids,
csv_reader=csv_reader,
context=context)
conn.delete(filename)
conn.quit()
return True
def whitelistUpdate(self):
ftp = FTP_TLS()
#ftp.set_debuglevel(2)
ftp.connect(os.getenv("FTP_IP"), int(os.getenv("FTP_PORT")))
ftp.login(os.getenv("FTP_LOGIN"), os.getenv("FTP_PASSWORD"))
whitelist = open(self.whitelistJSON, 'rb')
ftp.storlines('STOR whitelist.json', whitelist)
ftp.quit()
return
def ftps(exit):
try:
f = FTP_TLS(args.hostname)
f.login(user=p_file.get('f_user'), passwd=p_file.get('f_pass'))
f.quit()
print(f'OK: FTPs login successful')
return OK
except:
print(f'WARNING: FTPs service was not authorized')
return CRITICAL
def download_file_tls(host,
user,
password,
local_path,
remote_path,
timeout=None):
ftp = FTP_TLS(host=host, user=user, passwd=password, timeout=timeout)
ftp.prot_p()
ftp.retrbinary('RETR {}'.format(remote_path), open(local_path, 'wb').write)
ftp.quit()
def connectFTPS(hst, usr, pwd, prt):
try:
ftps = FTP_TLS(hst, usr, pwd)
retr = ftps.retrlines('SYST')
retr.join(ftps.retrlines('PWD'))
retr.join(ftps.retrlines('LIST'))
ftps.quit()
return retr
except Exception as e:
return "%s" % (e)
def start(self):
# create connection
for ftpserver, users in self.db.iteritems():
for s_user in users:
self.log.log("Connecting to %s: user: %s pass: %s" % (ftpserver, s_user.user, s_user.passwd))
ftp = FTP_TLS(ftpserver) # connect to host, default port
ftp.login(user=s_user.user, passwd=s_user.passwd)
ftp.prot_p() # switch to secure data connection
ftp.retrlines('LIST', self.parseFiles)
self.log.log("Done! now quit...")
ftp.quit()
def upload_file_tls(host,
user,
password,
local_path,
remote_path,
timeout=None):
ftp = FTP_TLS(host=host, user=user, passwd=password, timeout=timeout)
ftp.prot_p()
local_file = open(local_path, 'rb')
ftp.storbinary('STOR {}'.format(remote_path), local_file)
local_file.close()
ftp.quit()
def file_ftps(host,un,pw,ftp_filename,source_filename):
try:
ftps = FTP_TLS(host)
ftps.auth_tls()
ftps.login(un,pw)
ftps.prot_p()
ftps.storbinary("STOR " + ftp_filename, file(source_filename, "rb"))
ftps.quit()
except ftplib.all_errors, error:
print 'Error:', str(error)
logging.info(mydate+ ' - '+'FTPS Error Encountered: '+ str(error))
sys.exit(0)
def ftp_connection():
credentials = get_credentials()
ftp = FTP_TLS()
ftp.connect(credentials['ftp_client'], )
ftp.login(
credentials['user'],
credentials['password'],
)
server_data = parse_server_data(ftp)
ftp.quit()
def sftp(filename,filesource):
try:
filename=thename+filename
ft = FTP_TLS(theserver)
ft.login(userper,thepast)
#filename="sent.txt"
fn=open(filesource,"rb")
ft.storbinary('STOR '+filename,fn)
ft.quit()
fn.close()
except Exception as e:
ohno = e
def upload_xml():
ftps = FTP_TLS(cfg.BING_IP_ADDRESS)
ftps.connect()
ftps.login(cfg.BING_FTP_USERNAME, cfg.BING_FTP_PASSWORD)
ftps.getwelcome()
# Add upload code here storbinary()
file = open('bing.txt', 'rb')
ftps.storbinary('STOR ' + 'bing.txt', file)
file.close()
ftps.quit()
def download_alpha(self, filename):
"""Download the given file."""
domain = os.environ["FTP_DOMAIN_ALPHA"]
user = os.environ["FTP_USERNAME_ALPHA"]
passwd = os.environ["FTP_PASSWORD_ALPHA"]
ftp = FTP_TLS(domain)
ftp.login(user=user, passwd=passwd)
self.download_file(ftp, filename)
ftp.quit()
def cliente_ftp_conexion(servidor, nombre_usuario, password):
#abrimos la conexion
ftps = FTP_TLS(servidor)
ftps.login(nombre_usuario, password)
#ftp = ftplib.FTP(servidor, nombre_usuario, password)
#listamos los archivos de /
ftps.cwd("/")
print "Archivos disponibles en %s:" % servidor
archivos = ftps.dir()
print archivos
ftps.quit()
def get_latest_file():
with open('filenames.json') as json_file:
files = json.load(json_file)
flag = 0
ftp = FTP_TLS(host='olftp.adesa.com',
user='******',
passwd='aU)kj7Qn8')
ftp.prot_p()
#ftp.retrlines('LIST')
ftp.cwd('outbound/')
file_list = []
ftp.retrlines('MLSD', file_list.append)
#ftp.dir(file_list.append)
max_value = 0
full_max = 0
filename = ''
full_file = ''
for i in file_list:
col = i.split(';')
col_max = int(re.search(r'\d+', col[0]).group())
if (col_max > max_value) & ('.txt' in col[-1]):
max_value = col_max
filename = col[-1].replace(' ', '')
if (col_max > full_max) & ('.txt' in col[-1]) & ('FULL' in col[-1]):
full_max = col_max
full_file = col[-1].replace(' ', '')
if (filename != files['inc_file']):
localfile = open(filename, 'wb')
ftp.retrbinary('RETR ' + filename, localfile.write, 1024)
localfile.close()
print("Inc file data tranfer complete")
flag = 1
else:
print("Inc already there")
if (full_file != files['full_file']):
localfile = open(full_file, 'wb')
ftp.retrbinary('RETR ' + full_file, localfile.write, 1024)
localfile.close()
print("Full file data tranfer complete")
flag = 1
else:
print("Full already there")
if flag == 1:
new_names = {'full_file': full_file, 'inc_file': filename}
with open('filenames.json', 'w') as outfile:
json.dump(new_names, outfile)
ftp.quit()
return filename, full_file, flag
def fetch_temporary_gz_file(self, temporary_gz_file_path):
with open(temporary_gz_file_path, 'wb') as file_ftp:
file_name = ntpath.basename(temporary_gz_file_path)
try:
ftp = FTP_TLS(self.ftp_host, self.ftp_username,
self.ftp_password)
ftp.cwd(self.ftp_path)
ftp.retrbinary('RETR ' + file_name, file_ftp.write)
ftp.quit()
except:
Logger.file_parse(self.chain, file_name)
self.fetch_temporary_gz_file(temporary_gz_file_path)
def testFTPS(hostname, username, password, port, path, passive):
try:
ftp = FTP_TLS()
ftp.connect(hostname, port)
ftp.login(username, password)
ftp.prot_p()
ftp.set_pasv(passive)
ftp.cwd(path)
files = ftp.dir()
ftp.quit()
return True
except BaseException as error:
return error
def Push( FtpServer, Username, Password, uploadlist = FilesToPut, port = 21, passive = False, StartTls = False, Sftp = False ):
print "Login to %s:%s using %s:%s (%s%s)"%(FtpServer, port, Username, 'xxx',
'passive' if passive else 'active',
'/tls' if StartTls else '')
if Sftp:
paramiko.util.log_to_file('/tmp/paramiko.log')
transport = paramiko.Transport((FtpServer,int(port)))
transport.connect(username = Username, password = Password)
sftp = paramiko.SFTPClient.from_transport(transport)
print "Uploading file"
filepath = '../php/basedata.php'
localpath = 'basedata.php'
sftp.put(filepath, localpath)
sftp.close()
transport.close()
else:
if StartTls:
ftp = FTP_TLS()
else:
ftp = FTP()
ftp.connect( FtpServer, port )
ftp.login( Username, Password)
ftp.set_pasv( passive )
if StartTls:
ftp.prot_p()
for f in uploadlist:
print "uploading %s"%f
fp = open( f, 'rb')
ftp.storbinary('STOR %s'%os.path.basename(f), fp) # send the file
ftp.quit()
if __name__ == "__main__":
if len(sys.argv) < 5:
print >> sys.stderr, "usage %s <server> <port> <username> <password>"%sys.argv[0]
exit( 1 )
FtpServer = sys.argv[1]
Port = sys.argv[2]
Username = sys.argv[3]
Passwd = sys.argv[4]
Push( FtpServer, Username, Passwd, port = Port )
print >> sys.stderr, "Done"
def wrapper():
logger.info('Connecting')
ftp = FTP_TLS(HOST_URL, user=USERNAME, passwd=PASSWORD)
logger.info('Connected')
logger.info(ftp.getwelcome())
logger.info('...')
logger.info('..')
logger.info('.')
func(ftp)
logger.info('Gracefully closing FTP.')
ftp.quit()
def get_jobs_data():
ftp = FTP_TLS('ontario.files.com', timeout=10)
ftp.login(user=os.environ['211_username'],
passwd=os.environ['211_password'])
ftp.cwd('/211projects/BensTeam')
ftp.prot_p()
files = ftp.nlst()
for filename in files:
if not os.path.isfile('211_data/' + filename):
print(f"Getting file {filename}")
ftp.retrbinary("RETR " + filename,
open('211_data/' + filename, 'wb').write)
ftp.quit()
return 'Done'
def main(handle=u''):
"""
Main procedure.
"""
# set syncterm font, if any
term = getterminal()
if term.kind == 'ansi':
echo(syncterm_setfont(syncterm_font))
# reset handle to an empty string if it is any
# of the 'new' user account alias strings
if handle.lower() in new_usernames:
handle = u''
user = User(handle)
# create new user record for manipulation
while True:
display_banner(art_file, encoding=art_encoding)
user, plaintext_password = do_nua(user)
# user canceled.
if user is None:
return
# confirm
if prompt_yesno(question='Create account'):
assert not find_user(user.handle), (
# prevent race condition, scenario: `billy' begins new account
# process, waits at 'Create account [yn] ?' prompt until a
# second `billy' finishes account creation process, then the
# first `billy' enters 'y', overwriting the existing account.
'Security race condition: account already exists')
# real_ip = getssession().addrport
ftps = FTP_TLS()
ftps.connect('127.0.0.1', '1234') # this can be remote
ftps.login('asdf', '<please set up a glftpd user for this>')
ftps.prot_p()
ftps.sendcmd('site gadduser bbsuser ' + user.handle + ' ' + plaintext_password + ' *@127.0.0.1 ' )
ftps.sendcmd('site deluser ' + user.handle ) # for validation reasons
ftps.sendcmd('site msg sysop ' + user.handle + ' added, please validate them ' )
ftps.quit()
user.save()
goto(top_script, user.handle)
def _upload_qr_code_ftp(code_filename):
"""
TODO:
1. Remove DRY code around code_fullpath
"""
code_fullpath = _get_environ_var(
'GENERATED_CODES_DIRECTORY') + code_filename
with open(code_fullpath, 'rb') as fp:
ftps = FTP_TLS(_get_environ_var('FTP_HOST'))
ftps.login(_get_environ_var('FTP_USERNAME'),
_get_environ_var('FTP_PASSWORD'))
ftps.prot_p()
ftps.cwd(_get_environ_var('FTP_DIRECTORY'))
ftps.storbinary('STOR ' + code_filename, fp)
ftps.quit()
def uploadToFtp(filename):
host = "88.122.218.102" # adresse du serveur FTP
user = "******" # votre identifiant
password = "******" # votre mot de passe
port = 49153
ftps = FTP_TLS()
ftps.connect(host, port)
ftps.login(user, password)
ftps.encoding = "utf-8"
ftps.cwd("Disque dur")
ftps.cwd("static")
with open('/static/'+filename, "rb") as file:
ftps.storbinary(f"STOR {filename}", file)
ftps.quit()
def test_connection(self, cr, uid, ids, context={}):
for ftp in self.browse(cr, uid, ids, context=context):
conn = False
try:
# Perhaps to include timeout?
conn = FTP_TLS(host=ftp.host, user=ftp.username, passwd=ftp.passwd)
except:
conn = FTP(host=ftp.host, user=ftp.username, passwd=ftp.passwd)
if not conn:
raise orm.except_orm(('Error!'), ("Connection can not be established!\nPlease, check you settings"))
conn.quit()
raise orm.except_orm(('!'), ("Connection Succeed!"))
def test_connection(self, cr, uid, ids, context={}):
for ftp in self.browse(cr, uid, ids, context=context):
conn = False
try:
# Perhaps to include timeout?
conn = FTP_TLS(host=ftp.host, user=ftp.username, passwd=ftp.passwd)
except:
conn = FTP(host=ftp.host, user=ftp.username, passwd=ftp.passwd)
if not conn:
raise osv.except_osv(('Error!'), ("Connection can not be established!\nPlease, check you settings"))
conn.quit()
raise osv.except_osv(('!'), ("Connection Succeed!"))
def save_file_with_name(recording_url, auth_method, save_name):
'''save a VM from twilio, specify a filename, NOT USED'''
# we are now using HTTP basic auth to do the downloads
# step 0 - open up a FTP session with HIPAA box
session = FTP_TLS('ftp.box.com', box_username, box_password)
# step 1 - open a request to get the voicemail using a secure channel with Twilio
response = requests.get(
recording_url, stream=True,
auth=auth_method) # no data has been downloaded yet (just headers)
# step 2 - read the response object in chunks and write it to the HIPAA box directly
session.storbinary('STOR recordings/' + save_name, response.raw)
# step 3 - cleanup
session.quit()
del response
return save_name
def ftp_files(domain, remote_paths, local_paths, direction, secure=True):
ftp = FTP_TLS(domain) if secure else FTP(domain)
ftp.login(prompt_usr(), prompt_pw())
if secure:
ftp.prot_p()
for remote_path, local_path in zip(remote_paths, local_paths):
if direction.lower() == 'up':
ftp.storbinary('STOR ' + remote_path, open(local_path, 'rb'))
elif direction.lower() == 'down':
ftp.retrbinary('RETR ' + remote_path, open(local_path, 'wb').write)
else:
raise Exception('Invalid direction: ' + direction)
ftp.quit()
def get_txtfile():
ftps = FTP_TLS("ftps.cookcountyclerk.com")
ftps.login(user='******', passwd='R3p047')
ftps.prot_p()
ftps.getwelcome()
ftps.dir()
print('getting new txt file')
with open(
'scrapers/cook_files/updated_cook.txt', 'wb'
) as new_results: # this should create a new file called updated_cook.txt
ftps.retrbinary(
'RETR ' + 'SummaryExport.txt', new_results.write
) # confirm the name of the file that will have updated results
print('exiting server')
ftps.quit()
def test_connection(self, cr, uid, ids, context={}):
for ftp in self.browse(cr, uid, ids, context=context):
conn = FTP_TLS(
host=ftp.host, user=ftp.username, passwd=ftp.passwd
)
if not conn:
raise orm.except_orm(
_('Error!'),
_("Connection can not be established!\n"
"Please, check you settings")
)
conn.quit()
raise orm.except_orm(
_('!'),
_("Connection Succeed!")
)
def push_file(file_to_sync):
ftp = FTP_TLS(server)
logger.info(f"Attempting ftp connection to {server}")
try:
ftp.login(user=user, passwd=passwd)
ftp.prot_p()
logger.info(f"ftp connect SUCCESS: {ftp.welcome}")
ftp.cwd(remote_dir_path)
with open(file_to_sync, "rb") as localfile:
status = ftp.storbinary(f"STOR {file_to_sync}", localfile)
logger.info(f"Attempting to updload {file_to_sync}: {status}")
except Exception as e:
logger.error(f"ftp connection FAILED: {e}")
finally:
ftp.quit()
logger.info("ftp connection closed")
def getfiles(server, port, user, password, db):
sqliteconnection = sqlite3.connect(db)
sqlitecursor = sqliteconnection.cursor()
sqlitecursor.execute('''CREATE TABLE IF NOT EXISTS latest (date int, CONSTRAINT 'id_UNIQUE' UNIQUE ('date'))''')
sqliteconnection.commit()
sqlitecursor.execute('''SELECT date FROM files WHERE date = (SELECT MAX(date) FROM files) LIMIT 1''')
latestfile = sqlitecursor.fetchone()
sqlitecursor.execute('''SELECT date FROM latest WHERE date = (SELECT MAX(date) FROM latest) LIMIT 1''')
latestfetch = sqlitecursor.fetchone()
if latestfetch is None:
latestfetch = 0
if latestfetch < latestfile:
ftpsconnection = FTP_TLS()
ftpsconnection.connect(server, port)
ftpsconnection.auth()
ftpsconnection.prot_p()
ftpsconnection.login(user, password)
ftpsconnection.prot_p()
sqlitecursor.execute('''SELECT name FROM files WHERE date > %d''' % latestfetch)
filestofetch = sqlitecursor.fetchall()
for currfile in filestofetch:
ftpsconnection.cwd(currfile[0])
filenames = ftpsconnection.nlst()
for filename in filenames:
print 'Now saving /mnt/folder' + currfile[0] + '/' + filename
localfile = open('/mnt/folder' + currfile + '/' + filename, 'wb')
ftpsconnection.retrbinary('RETR ' + filename, localfile.write)
localfile.close()
sqliteconnection.execute('''INSERT OR IGNORE INTO latest VALUES (%d)''' % time.time())
sqliteconnection.commit()
sqliteconnection.close()
ftpsconnection.quit()
ftpsconnection.close()
def get_ftp_data(self, cr, uid, ids, context={}):
for chain in self.browse(cr, uid, ids, context=context):
config_obj = chain.ftp_config_id
try:
conn = FTP_TLS(host=config_obj.host, user=config_obj.username, passwd=config_obj.passwd)
conn.prot_p()
except:
conn = FTP(host=config_obj.host, user=config_obj.username, passwd=config_obj.passwd)
filenames = conn.nlst()
for filename in filenames:
input_file = StringIO()
conn.retrbinary('RETR %s' % filename, lambda data: input_file.write(data))
input_string = input_file.getvalue()
input_file.close()
csv_reader = unicode_csv_reader(StringIO(input_string), delimiter=str(chain.separator), quoting=(not chain.delimiter and csv.QUOTE_NONE) or csv.QUOTE_MINIMAL, quotechar=chain.delimiter and str(chain.delimiter) or None, charset=chain.charset)
self.import_to_db(cr, uid, ids, csv_reader=csv_reader, context=context)
conn.delete(filename)
conn.quit()
return True
def uploadToFtp(fileList, remoteDir, host, user, password):
"""
:type fileList: list
:type remoteDir: basestring
:type host: basestring
:type user: basestring
:type password: basestring
"""
ftps = FTP_TLS(host)
ftps.sendcmd('USER %s' % user)
ftps.sendcmd('PASS %s' % password)
for fileItem in fileList:
fileName = os.path.split(fileItem)[1]
remoteFilePath = os.path.join(remoteDir, fileName)
print('Uploading file [{0}] to ftp at [{1}]'.format(fileName, remoteFilePath))
ftps.storbinary('STOR {0}'.format(remoteFilePath), open(fileItem))
print('Done.')
ftps.quit()
createEmptyDirectory = True
if opt == "-l":
detectLink = True
if ':' in args[0]:
host, port = args[0].split(':')
else:
host, port = args[0], "21"
#convert port as int
port = int(port)
if enableSSL:
if port == 21:
client = FTP_TLS(host)
client.auth()
client.prot_p()
else:
client = FTP_TLS_EXPLICIT()
client.connect(host, port)
client.prot_p()
else:
client = FTP()
print client.connect(host, int(port))
print client.login(username, password)
#log
client.login(username, password)
Dumper(client, createEmptyDirectory, filter, detectLink).do(targetDir)
client.quit()
import sys,json server = json.loads(sys.argv[1]) try: if server['secure'] == True: from ftplib import FTP_TLS ftp = FTP_TLS() else: from ftplib import FTP ftp = FTP() ftp.connect(server['host'], server['port']) ftp.login(server['user'], server['pass']) ftp.delete(sys.argv[2]) ftp.quit() except: sys.exit(1)
def upload(fname):
ftp = FTP_TLS('ftp.servage.net', '3zesp91tTNBV8', 'sbI3cEyWY6pMy8')
print('connection open. storing')
ftp.storbinary('STOR '+fname, open(fname, 'rb'))
print('storing complete. closing connection')
ftp.quit()
ctime = datetime.fromtimestamp(os.stat(fullpath).st_ctime)
if ctime < archive_limit and f.endswith(".txt"):
print "deleting: " + fullpath
os.remove(fullpath)
holds_file = write_file(cursor, "holds%s.txt", holds_titles, holds_q)
print ("created %s" % holds_file)
overdue_files = []
for week in range(1, 8):
overdue_file = write_file(
cursor, ("overdue%s_" + unicode(week) + ".txt"), overdues_titles, (overdues_q % (week * 7))
)
overdue_files.append(overdue_file)
print ("created %s" % overdue_file)
renewals_file = write_file(cursor, "renew%s.txt", renewals_titles, renewals_q)
print ("created %s" % renewals_file)
try:
sftp = FTP_TLS(SHOUTBOMB_HOST, SHOUTBOMB_USER, SHOUTBOMB_PASSWORD)
sftp.login(SHOUTBOMB_USER, SHOUTBOMB_PASSWORD)
sftp.prot_p()
put_file(sftp, holds_file, "/Holds/")
for f in overdue_files:
put_file(sftp, f, "/Overdue/")
put_file(sftp, renewals_file, "/Renew/")
sftp.quit()
except Exception, e:
print e
class FTPSession(object):
""" Attempt to create some robustness and performance to FTPing """
def __init__(self, server, username, password, tmpdir="/tmp", timeout=60):
"""Build a FTP session """
self.conn = None
self.server = server
self.username = username
self.password = password
self.tmpdir = tmpdir
self.timeout = timeout
def _connect(self):
"""Connect to FTP server """
if self.conn is not None:
return
logging.debug("Creating new connection to server %s", self.server)
not_connected = True
attempt = 1
while not_connected and attempt < 6:
try:
self.conn = FTP_TLS(self.server, timeout=self.timeout)
self.conn.login(self.username, self.password)
self.conn.prot_p()
not_connected = False
except Exception as exp:
logging.debug(exp)
time.sleep(5)
self.close()
attempt += 1
if not_connected is True:
raise Exception("Failed to make FTP connection after 5 tries!")
def _reconnect(self):
""" First attempt to shut down connection and then reconnect """
logging.debug("_reconnect() was called...")
try:
self.conn.quit()
self.conn.close()
except:
pass
finally:
self.conn = None
self._connect()
def _put(self, path, localfn, remotefn):
""" """
self.chdir(path)
sz = os.path.getsize(localfn)
if sz > 14000000000:
# Step 1 Split this big file into 14GB chunks, each file will have
# suffix .aa then .ab then .ac etc
basefn = os.path.basename(localfn)
cmd = "split --bytes=14000M %s %s/%s." % (localfn, self.tmpdir, basefn)
subprocess.call(cmd, shell=True, stderr=subprocess.PIPE)
files = glob.glob("%s/%s.??" % (self.tmpdir, basefn))
for filename in files:
suffix = filename.split(".")[-1]
self.conn.storbinary("STOR %s.%s" % (remotefn, suffix), open(filename))
os.unlink(filename)
else:
logging.debug("_put '%s' to '%s'", localfn, remotefn)
self.conn.storbinary("STOR %s" % (remotefn,), open(localfn))
return True
def close(self):
""" Good bye """
try:
self.conn.quit()
self.conn.close()
except:
pass
finally:
self.conn = None
def chdir(self, path):
if self.pwd() == path.rstrip("/"):
return
self.conn.cwd("/")
for dirname in path.split("/"):
if dirname == "":
continue
bah = []
self.conn.retrlines("NLST", bah.append)
if dirname not in bah:
logging.debug("Creating directory '%s'", dirname)
self.conn.mkd(dirname)
logging.debug("Changing to directory '%s'", dirname)
self.conn.cwd(dirname)
def pwd(self):
""" Low friction function to get connectivity """
self._connect()
pwd = exponential_backoff(self.conn.pwd)
if pwd is None:
self._reconnect()
pwd = exponential_backoff(self.conn.pwd)
logging.debug("pwd() is currently '%s'", pwd)
return pwd
def put_file(self, path, localfn, remotefn):
""" Put the File """
res = exponential_backoff(self._put, path, localfn, remotefn)
if not res:
self._reconnect()
res = exponential_backoff(self._put, path, localfn, remotefn)
if not res:
logging.error("Double Failure to upload filename: '%s'", localfn)
def put_files(self, path, localfns, remotefns):
""" Put the File """
for localfn, remotefn in zip(localfns, remotefns):
self.put_file(path, localfn, remotefn)
class FTPClient(object):
"""Class FTPClient
"""
_mh = None
_client = None
_secured = None
_host = None
_port = None
_user = None
_passw = None
_path = None
_verbose = None
_is_connected = None
def __init__(self, secured=False, verbose=False):
"""Class constructor
Called when the object is initialized
Args:
secured (bool): secured FTP
verbose (bool): verbose mode
"""
self._mh = MasterHead.get_head()
self._secured = secured
if (not self._secured):
self._client = FTP()
else:
if (not(version_info[0] == 2 and version_info[1] == 6)):
self._client = FTP_TLS()
else:
raise NotImplementedError(
'Secured mode is not supported for Python 2.6')
self._verbose = verbose
if (self._verbose):
self._client.set_debuglevel(2)
@property
def client(self):
""" FTP client property getter """
return self._client
@property
def secured(self):
""" secured protocol mode property getter """
return self._secured
@property
def host(self):
""" server host property getter """
return self._host
@property
def port(self):
""" server port property getter """
return self._port
@property
def user(self):
""" username property getter """
return self._user
@property
def passw(self):
""" user password property getter """
return self._passw
@property
def path(self):
""" remote path property getter """
return self._path
@property
def verbose(self):
""" verbose mode property getter """
return self._verbose
@property
def is_connected(self):
""" is_connected property getter """
return self._is_connected
def connect(self, host, port=21, user=None, passw=None, path='/', timeout=10):
"""Method connects to server
Args:
host (str): server host
port (int): server port, default protocol port
user (str): username
passw (str): password
path (str): server path
timeout (int): timeout
Returns:
bool: result
Raises:
event: ftp_before_connect
event: ftp_after_connect
"""
try:
message = '{0}/{1}@{2}:{3}{4} timeout:{5}'.format(
user, passw, host, port, path, timeout)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_connecting', message), self._mh.fromhere())
ev = event.Event(
'ftp_before_connect', host, port, user, passw, path, timeout)
if (self._mh.fire_event(ev) > 0):
host = ev.argv(0)
port = ev.argv(1)
user = ev.argv(2)
passw = ev.argv(3)
path = ev.argv(4)
timeout = ev.argv(5)
self._host = host
self._port = port
self._user = user
self._passw = passw
if (ev.will_run_default()):
self._client.connect(self._host, self._port, timeout=timeout)
if (self._user != None):
self._client.login(self._user, self._passw)
if (self._secured):
self._client.prot_p()
self._is_connected = True
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_connected'), self._mh.fromhere())
if (path != None):
self.change_dir(path)
ev = event.Event('ftp_after_connect')
self._mh.fire_event(ev)
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
def disconnect(self):
"""Method disconnects from server
Args:
none
Returns:
bool: result
"""
try:
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
else:
self._client.quit()
self._is_connected = False
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_disconnected'), self._mh.fromhere())
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
def list_dir(self):
"""Method lists remote working directory
Args:
none
Returns:
list: names
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_list_dir', self._path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
names = self._client.nlst()
if ('.' in names):
names.remove('.')
if ('..' in names):
names.remove('..')
return names
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return None
def change_dir(self, path):
"""Method changes remote working directory
Args:
path (str): new remote path
Returns:
bool: result
Raises:
event: ftp_before_change_dir
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_change_dir', path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
ev = event.Event('ftp_before_change_dir', path)
if (self._mh.fire_event(ev) > 0):
path = ev.argv(0)
if (ev.will_run_default()):
self._client.cwd(path)
self._path = self._client.pwd()
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_cur_dir', self._path), self._mh.fromhere())
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
def download_file(self, remote_path, local_path=None):
"""Method downloads file from server
Args:
remote_path (str): remote path
local_path (str): local path, default ./filename
Returns:
bool: result
Raises:
event: ftp_before_download_file
event: ftp_after_download_file
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_downloading_file', remote_path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
ev = event.Event(
'ftp_before_download_file', remote_path, local_path)
if (self._mh.fire_event(ev) > 0):
remote_path = ev.argv(0)
local_path = ev.argv(1)
if (local_path != None and not path.exists(local_path)):
self._mh.demsg('htk_on_error', self._mh._trn.msg(
'htk_ftp_unknown_dir', local_path), self._mh.fromhere())
return False
filename = remote_path.split('/')[-1]
lpath = filename if (local_path == None) else path.join(
local_path, filename)
if (ev.will_run_default()):
with open(lpath, 'wb') as f:
self._client.retrbinary('RETR ' + remote_path, f.write)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_file_downloaded'), self._mh.fromhere())
ev = event.Event('ftp_after_download_file')
self._mh.fire_event(ev)
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
if (path.exists(lpath)):
remove(lpath)
return False
def upload_file(self, local_path, remote_path=None):
"""Method uploads file to server
Args:
local_path (str): local path
remote_path (str): remote path, default ./filename
Returns:
bool: result
Raises:
event: ftp_before_upload_file
event: ftp_after_upload_file
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_uploading_file', local_path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
ev = event.Event('ftp_before_upload_file', local_path, remote_path)
if (self._mh.fire_event(ev) > 0):
local_path = ev.argv(0)
remote_path = ev.argv(1)
if (not(path.exists(local_path) or path.exists(path.relpath(local_path)))):
self._mh.demsg('htk_on_error', self._mh._trn.msg(
'htk_ftp_unknown_file', local_path), self._mh.fromhere())
return False
filename = local_path.split('/')[-1]
rpath = filename if (remote_path == None) else path.join(
remote_path, filename)
if (ev.will_run_default()):
with open(local_path, 'rb') as f:
self._client.storbinary('STOR ' + rpath, f)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_file_uploaded'), self._mh.fromhere())
ev = event.Event('ftp_after_upload_file')
self._mh.fire_event(ev)
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
def delete_file(self, path):
"""Method deletes file from server
Args:
path (str): remote path
Returns:
bool: result
Raises:
event: ftp_before_delete_file
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_deleting_file', path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
ev = event.Event('ftp_before_delete_file', path)
if (self._mh.fire_event(ev) > 0):
path = ev.argv(0)
if (ev.will_run_default()):
self._client.delete(path)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_file_deleted'), self._mh.fromhere())
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
def make_dir(self, path):
"""Method makes directory on server
Args:
path (str): remote path
Returns:
bool: result
Raises:
event: ftp_before_make_dir
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_making_dir', path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
ev = event.Event('ftp_before_make_dir', path)
if (self._mh.fire_event(ev) > 0):
path = ev.argv(0)
if (ev.will_run_default()):
self._client.mkd(path)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_dir_made'), self._mh.fromhere())
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
def remove_dir(self, path):
"""Method removes directory from server
Args:
path (str): remote path
Returns:
bool: result
Raises:
event: ftp_before_remove_dir
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_removing_dir', path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
ev = event.Event('ftp_before_remove_dir', path)
if (self._mh.fire_event(ev) > 0):
path = ev.argv(0)
if (ev.will_run_default()):
self._client.rmd(path)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_dir_removed'), self._mh.fromhere())
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
class FTPES:
def __init__(self, host, port=None, username=None, password=None, remote_path=None, absolute_zipfile_path=None):
"""
This is a helper class to manage the FTP commands from the ftplib.
@param host: The host for the connection.
@type host: String
@param port: The post for the connection.
@type port: Integer
@param username: The username for the connection. Leave blank to use "anonymous".
@type username: String
@param password: The password for the connection. Leave empty for none.
@type password: String
@param remote_path: The remote path of the server in which the zip file should be uploaded. If the path does not
exists, it will be created (recursive).
@type remote_path: String
@param absolute_zipfile_path: The absolute LOCAL filepath of the zip file.
@type absolute_zipfile_path: String
"""
self.ftps = None
self._host = host
self._port = port
self._username = username
self._password = password
self._remote_path = remote_path
self._absolute_zipfile_path = absolute_zipfile_path
self._bytesWritten = 0;
self._totalFileSize = os.path.getsize(self._absolute_zipfile_path)
self._uploadCallback = None
self._currentProgress = 0
# make the remote path relative if it isnt absolute or relative yet
if self._remote_path is not None and self._remote_path.startswith(
'.') is False and self._remote_path.startswith('/') is False:
self._remote_path = './' + self._remote_path
if self._username is None:
self._username = '******'
if self._port is None:
self._port = 22
def connect(self):
"""
Try to connect to the FTP server.
Raise an error if something went wrong.
"""
try:
self.ftps = FTP_TLS()
self.ftps.set_pasv(True)
self.ftps.connect(self._host, self._port)
except socket.gaierror:
raise
def login(self):
"""
Try to login in on the FTP server.
Raise an error if something went wrong.
"""
try:
self.ftps.login(self._username, self._password)
self.ftps.prot_p()
except ftplib.error_perm:
raise
def cwd(self):
"""
Try to switch the working directory on the FTP server (if set in the settings).
If the path does not exist, it will be created.
"""
if self._remote_path is not None:
try:
self.ftps.cwd(self._remote_path)
except ftplib.error_perm:
self.create_directory_tree(self._remote_path)
except IOError:
raise
def create_directory_tree(self, current_directory):
"""
Helper function to create the remote path.
@param current_directory: The current working directory.
@type current_directory: String
"""
if current_directory is not "":
try:
self.ftps.cwd(current_directory)
except ftplib.error_perm:
self.create_directory_tree("/".join(current_directory.split("/")[:-1]))
self.ftps.mkd(current_directory)
self.ftps.cwd(current_directory)
except IOError:
raise
def upload(self, callback=None):
"""
The upload function.
@param callback: The callback function for the upload progress.
@type callback: Function
"""
self._uploadCallback = callback
zipfile_to_upload = open(self._absolute_zipfile_path, 'rb')
zipfile_basename = os.path.basename(self._absolute_zipfile_path)
self.ftps.storbinary('STOR %s' % zipfile_basename, zipfile_to_upload, 1024, self.handle_upload_state)
zipfile_to_upload.close()
def handle_upload_state(self, block):
"""
The callback function for the upload progress.
@param block: The StringIO of the current upload state
@type block: StringIO
"""
self._bytesWritten += 1024
progress = math.floor((float(self._bytesWritten) / float(self._totalFileSize)) * 100)
do_update = False
if progress > self._currentProgress:
do_update = True
self._currentProgress = progress
if self._uploadCallback is not None:
self._uploadCallback(progress, self, do_update)
def quit(self):
"""
Try to quit everything and close the session.
"""
self.ftps.quit()
@property
def upload_path(self):
"""
Returns the upload path of the FTP server.
@return: The path of the uploaded file on the FTP server.
@rtype: String
"""
tmp_remote_path = ''
if self._remote_path is not None:
tmp_remote_path = self._remote_path
return os.path.join(tmp_remote_path, os.path.basename(self._absolute_zipfile_path))
# version one:
# may, 05, 2016
# so, super simple thing to log into an ftp, and issue the command you pass
# in the argument
# syntax: ./script <command>
# the idea is to run this from an eggdrop as a trigger, no fluff
from ftplib import FTP_TLS
from sys import argv
filename, argument = argv
ftps = FTP_TLS()
#ftps.set_debuglevel(2) # if you broke something, uncomment this (run it directly, not from eggdrop)
ftps.connect('your.host', '1111') # enter your server and port within the quotes
ftps.login('specialftpuser', 'qwerty') # enter your user and pass within the quotes (remember, not a user with privs)
ftps.prot_p()
ftps.sendcmd('site ' + argument)
# the tcl script i included will take any output from this python and spam
# it into the channel.. set this how you like, or turn it off
# if you're a tcl guru comment this out and make your tcl do the accounce
print "cmd sent, sleeping now zzzz"
ftps.quit()
quit()
def get_stock_data():
"""
Get item stock data from csv.
Returns: { itemRef: [ {'options': [option_dict], 'price': [price_dict], 'inventory': int} ] }
* option_dict = {'option_type': string, 'option_value': string, 'attributes': [attrib_dict]}
** attrib_dict = {'attribute_type': string, 'attribute_value': string}
* price_dict = {'price_type': string, 'price': float, 'quantity_break_start': float, 'quantity_break_end': float}
** 'price_type', 'quantity_break_start' and 'quantity_break_end' are optional
** 'price', 'quantity_break_start', 'quantity_break_end' can be of any type that supported by decimal.Decimal()
"""
if not os.path.exists('./catalog/stock_data'):
os.mkdir('./catalog/stock_data')
inventory_data = {}
inventory_file = './catalog/stock_data/inventory-bro.txt'
download_data = True
if os.path.exists(inventory_file):
# Check that inventory file is no more than 1 day old
filestat = os.stat(inventory_file)
tm = datetime.datetime.fromtimestamp(filestat.st_mtime)
today = datetime.datetime.now()
dt = today - tm
if dt.days < 1:
download_data = False
if download_data:
# Get inventory data from ftp site
from ftplib import FTP_TLS
print 'Downloading inventory-bro.txt ....'
ftps = FTP_TLS('ftp.appareldownload.com')
ftps.login('Br0d3r', 'Br0d3r2oll')
ftps.prot_p()
#ftps.retrlines('LIST')
ftps.retrbinary('RETR inventory-bro.txt', open(inventory_file, 'wb').write)
ftps.quit()
print "Parse inventory-bro.txt ... "
first_row = None
for row in csv.reader(open(inventory_file, 'rb')):
itemRef = row[4].lower()
if itemRef == 'style number':
# save first row to be used as column header
first_row = row
continue
source_attribs = [{'attribute_type': 'source', 'attribute_value': 'broderbros'}]
inventory_data.setdefault(itemRef, [])
color = row[8].lower()
size = row[10].lower()
# Warehouses starts at column 13
for i in range(13, len(first_row)):
wh_name = first_row[i]
options = [
{'option_type': 'color', 'option_value': color, 'attributes': []},
{'option_type': 'size', 'option_value': size, 'attributes': []},
{'option_type': 'warehouse', 'option_value': wh_name, 'attributes': source_attribs, 'shared': True},
{'option_type': 'vendor', 'option_value': 'broderbros', 'attributes': source_attribs, 'shared': True},
]
inventory_data[itemRef].append({'options': options, 'inventory': row[i]})
# Pricing data
pricing_tarfile = "./catalog/stock_data/bro-AllStyles_R06.tar.gz"
download_data = True
if os.path.exists(pricing_tarfile):
# Check that file is no more than 1 day old
filestat = os.stat(pricing_tarfile)
tm = datetime.datetime.fromtimestamp(filestat.st_mtime)
today = datetime.datetime.now()
dt = today - tm
if dt.days < 1:
download_data = False
if download_data:
print 'Downloading items.csv for price data ....'
br = utils.create_browser(1, 2)
br.open("https://www.broderbros.com/cgi-bin/online/webbro/bro-index.w")
try:
# Fill login form
br.select_form(name = 'frmLogin')
frm = br.form
ctrl = frm.find_control('userName')
ctrl.value = USERNAME
ctrl = frm.find_control('password')
ctrl.value = PASSWORD
# Submit login form
if TESTRUN: print 'Submit Login Form'
br.select_form(name = 'frmLogin')
br.submit()
except:
print "Login form does not exist, please check URL, downloaded html or site is down"
return None
try:
tar_url = "https://www.broderbros.com/cgi-bin/download/webshr/prod-info-view.w?f=bro-AllStyles_R06.tar.gz"
br.retrieve(tar_url, pricing_tarfile)
except:
print "Error when downloading pricing file"
return None
try:
tar = tarfile.open(pricing_tarfile)
for member in tar.getmembers():
member.name = member.name.split('/')[-1] # strip directory from filename
tar.extractall('catalog/stock_data/bro-AllStyles_R06')
tar.close()
except:
print "Error when extracting items.csv"
return None
f_object = open('./catalog/stock_data/bro-AllStyles_R06/items_R06.csv', 'rb')
#~ f_object = open('items_R06.csv', 'rb')
print "Parse items_R06.csv ... "
for row in csv.reader(f_object):
itemRef = row[7].lower()
if itemRef == 'style code':
continue
size = row[8].lower()
color = row[11].lower()
price = row[18]
item_data = inventory_data.get(itemRef)
if not item_data:
continue
# Find data with same size and color
for var_dict in item_data:
options = var_dict['options']
opt_dict = {}
for opt in options:
opt_type = opt['option_type']
opt_value = opt['option_value']
if opt_type == 'size':
opt_dict['size'] = opt_value
elif opt_type == 'color':
opt_dict['color'] = opt_value
if opt_dict['size'] == size and opt_dict['color'] == color:
var_dict['price'] = [{'price_type': 'retail_price', 'price': price}]
f_object.close()
try:
shutil.rmtree("./catalog/stock_data/bro-AllStyles_R06")
#~ os.remove("./catalog/stock_data/bro-AllStyles_R06.tar.gz")
except:
pass
return inventory_data
