-
Notifications
You must be signed in to change notification settings - Fork 0
/
deploy.py
379 lines (352 loc) · 13.2 KB
/
deploy.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
"""
Script to deploy a website to the server by ftp.
- Compares local directory with remote directory
- Updates modified files
- Adds new files
- Optionally, removes deleted files from remote
Requires: python 3.3+
Due to use of ftplib.mlsd()
The MIT License (MIT)
Copyright (c) 2015 James Benson
"""
"""
TODO: FTP response codes to look out for:
- 502 unknown command
- 550 empty directory
- 451 can't remove directory
Good ones:
- 226 transfer complete
"""
asciiExt = ['coffee', 'css', 'erb', 'haml', 'handlebars', 'hb', 'htm', 'html',
'js', 'less', 'markdown', 'md', 'ms', 'mustache', 'php', 'rb', 'sass', 'scss',
'slim', 'txt', 'xhtml', 'xml'];
deleteIgnoreFiles = ["/.ftpquota"];
deleteIgnoreDirs = ["/cgi-bin"];
remoteSep = "/";
dLogName = "debug.txt";
STOR_AUTO = 0;
STOR_BINARY = 1;
STOR_ASCII = 2;
UPLOAD_OVERWRITE = 0;
UPLOAD_MODIFIED = 1;
######################### SETUP ##########################
remoteHost = "127.0.0.1";
remoteUser = "Benson";
remotePassword = "benson";
localPath = "D:\\test\\ftp";
remotePath = "/";
### OPTIONS ###
verbose = True;
remoteTLS = False; # SSL/TLS doesn't work invalid certificate error
remoteDelete = True;
remoteIgnoreHidden = False; # TODO: Implement hidden.
storMode = STOR_BINARY; # only binary currently works
uploadMode = UPLOAD_MODIFIED;
debug = True;
##########################################################
import os;
from datetime import datetime, timedelta;
from ftplib import FTP, FTP_TLS, error_reply, error_temp, error_perm, error_proto, all_errors;
if remoteTLS:
import ssl;
ftp = None;
dLog = None;
# === FTP Functions ===
def connect():
global ftp;
if remoteTLS:
context = ssl.create_default_context();
ftp = FTP_TLS(remoteHost, remoteUser, remotePassword, acct="", keyfile=None, certfile=None, context=context, timeout=20);
ftp.prot_p();
else:
ftp = FTP(remoteHost, remoteUser, remotePassword, 20);
print(ftp.getwelcome());
def stor(dirpath, file):
"""Store the file obj to the dirpath of server."""
ext = (os.path.splitext(file.name())[1]).lstrip('.');
storpath = remoteJoin(dirpath, file.name());
try:
if (storMode == STOR_ASCII) or (storMode == STOR_AUTO and ext in asciiExt):
# Store in ASCII mode
if verbose: print("[asc] ", end="");
ftp.storlines("STOR %s" % storpath, open(file.path));
else:
# Store in binary mode
if verbose: print("[bin] ", end="");
ftp.storbinary("STOR %s" % storpath, open(file.path, "rb"));
setModified(dirpath, file);
if verbose: print("Uploaded: %s -> %s" % (file.path, storpath));
except OSError as oserror:
print("Failed Upload: %s\n %s" % (file.path, oserror));
def setModified(dirpath, file):
"""Attempts to set the modified time with MFMT."""
ftp.voidcmd("MFMT %s %s" % (file.getModified(), remoteJoin(dirpath, file.name())));
def rm(dirpath, file):
"""Delete the file at the path from the server."""
p = remoteJoin(dirpath, file.name());
_rm(p);
if verbose: print("Deleted: %s" % p);
def _rm(filepath):
ftp.delete(filepath);
def mkDir(dirpath, name):
dirname = remoteJoin(dirpath, name);
ftp.mkd(dirname);
if verbose: print("Created: %s" % dirname);
def rmDir(dirpath, name, recursive = False):
dirname = remoteJoin(dirpath, name);
if recursive:
_rmDirR(dirname);
_rmDir(dirname);
else:
_rmDir(dirname);
if verbose: print("Deleted: %s" % remoteJoin(dirname, "*"));
def _rmDir(dirpath):
"""Delete directory with name from the current working directory.
Only deletes empty directories."""
ftp.rmd(dirpath); # TODO: What if fails to delete?
def _rmDirR(dirpath):
"""Remove the directory at dirpath and its contents (recursive)."""
try:
dirs, files = listRemote(dirpath);
for f in files:
_rm(f.path);
for d in dirs:
_rmDirR(d.path);
_rmDir(d.path);
except:
raise error_temp("451 Can't remove directory");
# === End FTP Functions ===
# === Traversal Functions ===
def traverse(localPath, remotePath = remoteSep):
dprint("TRAVERSING: local %s | remote %s"%(localPath, remotePath));
localDirs, localFiles = listLocal(localPath);
remoteDirs, remoteFiles = listRemote(remotePath);
newF, modifiedF, unmodifiedF, deletedF = compareFiles(localFiles, remoteFiles, remoteDelete);
newD, existingD, deletedD = compareDirs(localDirs, remoteDirs, remoteDelete);
for f in newF + modifiedF:
stor(remotePath, f);
for d in newD:
mkDir(remotePath, d);
for d in newD + existingD:
dname = d.name();
traverse(os.path.join(localPath, dname), remoteJoin(remotePath, dname));
if remoteDelete:
for d in deletedD:
rmDir(remotePath, d, True);
for f in deletedF:
rm(remotePath, f);
def listLocal(path):
dirs = [];
files = [];
names = os.listdir(path);
for n in names:
fullp = os.path.join(path, n);
if os.path.isdir(fullp):
dirs.append(Directory(fullp));
if os.path.isfile(fullp):
f = File(fullp);
f.setModifiedTimestamp(os.stat(fullp).st_mtime);
files.append(f);
return (dirs, files);
def listRemote(path = ""):
dirs = [];
files = [];
response = ftp.mlsd(path);
for name, fact in response:
if fact["type"] == "dir":
dirs.append(Directory(remoteJoin(path, name)));
if fact["type"] == "file":
f = File(remoteJoin(path, name));
f.setModifiedUTCStr(fact["modify"]);
files.append(f);
return (dirs, files);
# === End Traversal Functions ===
def remoteJoin(pathA, pathB):
if not pathA.endswith(remoteSep) and not pathB.startswith(remoteSep):
pathA += remoteSep;
elif pathA.endswith(remoteSep) and pathB.startswith(remoteSep):
pathA = pathA.rstrip(remoteSep);
return pathA + pathB;
# === Structures ===
class File(object):
def __init__(self, path):
self.path = str(path);
self.datetimeFormat = "%Y%m%d%H%M%S";
def __str__(self):
return self.name();
# Object Comparison
def __eq__(self, other):
"""As File objects will only be compared within a directory the unique
identitifier will be the name."""
if isinstance(other, File):
return self.name() == other.name();
else:
return self.name() == str(other);
def __lt__(self, other):
"""Determine if the file is older than other using the modified timestamp."""
return self.modified < other.modified;
def __gt__(self, other):
"""Determine if the file is newer than other using the modified timestamp."""
return self.modified > other.modified;
def __le__(self, other):
"""Determine if the file is older or the same than other using the modified timestamp."""
return self.modified <= other.modified;
def __ge__(self, other):
"""Determine if the file is newer or the same than other using the modified timestamp."""
return self.modified >= other.modified;
# End Object Comparison
def name(self):
return os.path.basename(self.path);
def setModifiedUTCStr(self, modified):
# Should be a string of the utc time.
self.modified = datetime.strptime(modified, self.datetimeFormat);
def setModifiedTimestamp(self, modified):
# Timestamp (in windows at least) gives extra microseconds (us) that ftp doesn't have
usModified = datetime.utcfromtimestamp(modified)
usExtra = timedelta(microseconds=usModified.microsecond);
self.modified = usModified - usExtra;
def getModified(self):
return datetime.strftime(self.modified, self.datetimeFormat);
class Directory(object):
def __init__(self, path):
self.path = path;
def __str__(self):
return self.name();
def __eq__(self, other):
if isinstance(other, Directory):
return self.name() == other.name();
else:
return self.name() == str(other);
# def __len__(self):
# len()
def name(self):
if isinstance(self.path, Directory):
raise Exception("Expected str found Directory");
return os.path.basename(self.path);
# === End Structures ===
def compareFiles(localList, remoteList, checkDeleted = True):
"""Compares localList with remoteList gets the tuple containing File objects:
(new, modified, unmodified, deleted)
new: Files that are in localList but not in remoteList.
modified: Files that are newer in localList than remoteList.
unmodified: Files that are the same in both lists.
deleted: Files that are in the remoteList but not in localList.
*newer is defined by the file's date modified attribute.
New, Modified and Unmodified will contain local files objects that need to
be uploaded to the remote location.
Deleted will contain remote file objects that need to be deleted from
the remote location."""
new = [];
modified = [];
unmodified = [];
deleted = [];
dprint("COMPARE FILES");
for lfile in localList:
dprint("LOCAL: %s - %s" % (lfile.path, lfile.modified));
existsInRemote = False;
for rfile in remoteList:
if lfile == rfile:
dprint("REMOTE: %s - %s" % (rfile.path, rfile.modified));
existsInRemote = True;
if uploadMode == UPLOAD_OVERWRITE or lfile > rfile:
dprint("Upload Mode: %s | Modified: lfile > rfile" % uploadMode);
modified.append(lfile);
else:
dprint("Not Modified: lfile <= rfile");
unmodified.append(lfile);
break;
if not existsInRemote:
dprint("New local file");
new.append(lfile);
dprint("--------------------------------------");
# Check for deleted files
if checkDeleted:
dprint("CHECK FOR DELETED FILES");
for rfile in remoteList:
existsInLocal = False;
for lfile in localList:
if rfile == lfile:
existsInLocal = True;
break;
if not existsInLocal and not rfile.path in deleteIgnoreFiles:
dprint("DELETED: %s" % rfile.path);
deleted.append(rfile);
dprint("--------------------------------------");
return (new, modified, unmodified, deleted);
def compareDirs(localList, remoteList, checkDeleted = True):
"""Compares localList with remoteList gets the tuple containing string
names of the directories: (new, existing, deleted)
new: Directories that are in localList but not in remoteList.
existing: Directories that are in both lists.
deleted: Directories that are in the remoteList but not in localList.
localList - list of strings of the directory names in the local location.
remoteList - list of strings of the directory name in the remote location."""
new = [];
existing = [];
deleted = [];
dprint("COMPARE DIRECTORIES");
for ldir in localList:
dprint("LOCAL DIR: %s"%ldir.path);
existsInRemote = False;
for rdir in remoteList:
if ldir == rdir:
dprint("REMOTE DIR: %s"%rdir.path);
dprint("Exists On Local and Remote");
existsInRemote = True;
existing.append(ldir)
break;
if not existsInRemote:
dprint("New Local Directory");
new.append(ldir);
# Check for deleted directories
if checkDeleted:
dprint("CHECK FOR DELETED DIRECTORIES");
for rdir in remoteList:
existsInLocal = False;
for ldir in localList:
if rdir == ldir:
existsInLocal = True;
break;
if not existsInLocal and not rdir.path in deleteIgnoreDirs:
dprint("DELETED: %s" % rdir.path);
deleted.append(rdir);
dprint("--------------------------------------");
return (new, existing, deleted);
def dprint(line, end="\n"):
global dLog;
if debug:
if dLog == None:
if os.path.exists(dLogName):
os.remove(dLogName);
dLog = open(dLogName, "w")
dLog.write(line + end);
def main():
if not os.path.isdir(localPath):
print("Path Not Found: %s" % localPath);
return -1;
try:
connect();
traverse(localPath, remotePath);
except error_reply as r:
print(r);
except error_temp as t:
print(t);
except error_perm as p:
print(p);
except error_proto as pr:
print(pr);
except all_errors as a:
# REVIEW: all_errors is a tuple of (Error, OSError, EOFError)
# printing like this won't work I doubt, but I'm doing it anyway.
print(a);
finally:
if not ftp == None:
try:
ftp.quit();
except: pass;
ftp.close();
if not dLog == None and not dLog.closed:
dLog.flush();
dLog.close();
if __name__ == "__main__":
main();