def create_md5_table(): out.log('calculating local md5 hashes', 'sync') files = [] md5_table = {} for root, dirnames, filenames in os.walk(engine.LOCAL_WWW_DIR): #for filename in fnmatch.filter(filenames, '*'): for filename in filenames: abs_file = os.path.join(root, filename) rel_file = abs_file[len(engine.LOCAL_WWW_DIR)+1:] if isinstance(rel_file, str): rel_file = unicode(rel_file, 'utf-8') if NORMALIZE_UNICODE_FILENAMES: rel_file = unicodedata.normalize(NORMALIZATION_FORM, rel_file) files.append(rel_file) for f in files: try: md5_table[f] = engine.md5sum(f) except IndexError: pass except IOError: if not (ignored_file(f) or system_file(f)): out.log('could not open ' + f, 'sync', out.LEVEL_WARNING) return md5_table
def load_md5_table(filename): out.log('loading hash table from ' + filename, 'sync') engine.sync_ftp() try: #load form file md5_table_file = open(filename, 'r', encoding='utf-8') md5_table = json.load(md5_table_file) md5_table_file.close() except: #if that fails: return empty table out.log('unable to load hashtable from ' + filename + ', creating empty table.', 'sync', out.LEVEL_WARNING) md5_table = {} #normalize unicode entries if NORMALIZE_UNICODE_FILENAMES: normalized_md5_table = {} for key in md5_table: if isinstance(key, str): key = unicode(key, 'utf-8') normalized_key = unicodedata.normalize(NORMALIZATION_FORM, key) normalized_md5_table[normalized_key] = md5_table[key] md5_table = normalized_md5_table #give back return md5_table
def copy_wp_config(when_exists='skip'): out.log('copy wp-config.php to wordpress directory', 'wordpress') #check if wp-config.php already exists if overwrite_allowed(engine.LOCAL_WP_DIR + '/wp-config.php', when_exists): #copy our config.php to the desired location run.local('cp php/wordpress/wp-config.php ' + engine.LOCAL_WP_DIR)
def get_compressed(remote_file, local_file=None, verbose=False, permissions=None, fast_compression=False, preserve_remote_file=False): import php #compress remote file compressed_remote = gzip.compress_remote(remote_file, fast=fast_compression) php.flush_buffer( ) #in case we were buffering php execution, it is time to execute now #download compressed_local = get(compressed_remote, gzip.compressed_filename(local_file), verbose, permissions) try: filesize = os.path.getsize(compressed_local) out.log('downloaded compressed ' + str(filesize / 1000.0) + ' kb file', 'transfer') except OSError: pass if preserve_remote_file: #restore uncompressed remote file gzip.uncompress_remote(compressed_remote) else: #or tell the engine it has been renamed engine.rename_remote_file(remote_file, compressed_remote) #uncmopress local file return gzip.uncompress_local(compressed_local, True)
def pack_remote_list(file_list): #tell out.log('packing a bunch of remote files', 'tar') #validate files if not validate_files_list(file_list): out.log('Error: files need to be given relative to www dir.', 'tar', out.LEVEL_ERROR) engine.quit() #register tar file tar_file = engine.get_new_remote_file('tar') #register list file list_file_content = "\n".join(file_list) list_file = engine.write_remote_file(list_file_content, 'files') #assemble packing command. Always pack from www dir pack_command = 'tar cf ' + tar_file + ' -C ' + engine.NORM_WWW_DIR + ' --files-from ' + list_file #pack! run.remote(pack_command) #and return packed file, of course return tar_file
def processModule(targetModule, doc_destination, pyPath=None, excluded_modules=[]): """ Processes a module given by a target directory path and generates documentation @input targetModule - the full path to the module to be documented. @input pyPath - if the python path must be updated to help import this module, make those changes. Defaults to the directory above the requested module. @input excluded_modules - a list of strings representing modules and packages that should not be documented. @return - a list of strings representing the names of the html files created in the documentation process. """ # process default values for vars that cannot be set to "None" if pyPath == None: pyPath = targetModule + "/.." # Map out all the modules to document names = doc.indexPythonModule(targetModule) out.log("Found " + str(len(names)) + " modules to document in module " + str(os.path.basename(targetModule)) ) # change to the destination directory so pydoc can do its thing os.chdir(doc_destination) out.debugPrint("Changed working directory to " + os.getcwd()) # Generate their documentation sys.path.append(os.path.abspath(pyPath)) filenames = doc.documentModules(names, exclude=excluded_modules, destination=doc_destination ) out.log("Wrote documentation for " + str(len(filenames)) + " modules into directory:\n\t" + str(doc_destination)) return filenames
def cleanup(namespace = None, local = True, remote = True): #cleanup all namespaces if namespace is None: #make a copy of the list before iterating over it up_for_delete_list = list(local_tmp_files) #cleanup every entry of this list for name in up_for_delete_list: cleanup(name, local, remote) #remove guard and exit cleaning_up_already = False return import transfer out.log('Removing tmp files in namespace ' + namespace, 'cleanup', out.LEVEL_DEBUG) #remove remote files first, because there removal might cause local files to happen if remote: for file in remote_tmp_files[namespace]: transfer.remove_remote(file) remote_tmp_files[namespace] = [] #then remove local files if local: for file in local_tmp_files[namespace]: transfer.remove_local(file) local_tmp_files[namespace] = []
def quit(): if not finalizing_in_process: finalize() out.log('Exited with errors. Look at output/output.log for a detailed log.', 'engine', out.LEVEL_ERROR) exit() else: out.log('An error occured during finalizing. We ignore it and try to finish finalization.', 'engine', out.LEVEL_ERROR)
def put_multiple(file_list): #split into ascii and non-ascii ascii_files, non_ascii_files = engine.split_by_encoding(file_list) if engine.FORCE_FTP_FILE_TRANSFER: ascii_files = [] non_ascii_files = file_list #pack ascii files, upload compressed and unpack on server if len(ascii_files) > 0: out.log('uploading files with ascii compatible names', 'transfer') local_tar = tar.pack_local_list(ascii_files) remote_tar = put_compressed(local_tar) tar.unpack_remote(remote_tar) #take the non-ascii files and upload them one after another using ftp if len(non_ascii_files) > 0: out.log('uploading files with non-ascii filename', 'transfer') directories = engine.get_all_directories(non_ascii_files) for directory in directories: create_remote_directory(directory) command = '' for f in non_ascii_files: command += u'put ' + engine.LOCAL_WWW_DIR + '/' + f + u' ' + ftp_path( f) + u'\n' ftp.execute(command)
def import_local_db(filename, compression=None): import os #is our file compressed? if compression is None: compression = gzip.is_compressed(filename) if compression: #uncompress sql_file = gzip.uncompress_local(filename) else: sql_file = filename #wipe old db if os.path.isfile(sql_file): truncate_local_db() else: out.log('File not found: ' + filename, 'mysql', out.LEVEL_ERROR) quit() #refill db out.log('importing local database from file ' + sql_file, 'mysql') execute_local_file(sql_file) #compress again, although it would have been better to not touch the compressed file in the first place if compression: gzip.compress_local(sql_file)
def remove_command_file(): out.log('removing command file', 'run') try: engine.NORM_COMMAND_FILE except: out.log("Error: Could not find NORM_COMMAND_FILE. No file to remove", 'run', out.LEVEL_WARNING) return transfer.remove_remote(engine.NORM_COMMAND_FILE)
def create_remote_directory(directory, permissions=None): out.log('create remote directory: ' + directory, 'transfer', out.LEVEL_VERBOSE) command = 'mkdir ' + ftp_path(directory) if engine.FTP_CLIENT == 'sftp': command = '-' + command if permissions is not None: command += "\nchmod " + str(permissions) + " " + ftp_path(directory) ftp.execute(command)
def initialize(command): import out import ftp out.clear_logfile() global enable_ftp_buffer if command in enable_ftp_buffer: out.log('enabling complete ftp buffering for ' + command + ' command', 'engine', out.LEVEL_DEBUG) ftp.start_buffer()
def remote_python_script(script_name, arguments = ''): #tell out.log('running python script ' + script_name, 'remote', out.LEVEL_VERBOSE) #upload script remote_script = transfer.put(script_name) #run remote('HOME=. python ' + remote_script + ' ' + arguments)
def decorated_func(*args, **kwargs): global start_time start_time = time.time() initialize(command) result = func(*args, **kwargs) finalize() elapsed_time = "{:.3f}".format(time.time() - start_time) out.log('Done. Took ' + elapsed_time + ' seconds.') return result
def assemble_htaccess_data(domain): ht_data = engine.read_local_file(engine.SCRIPT_DIR + '/htaccess/signature.htaccess') ht_data += engine.read_local_file(engine.SCRIPT_DIR + '/htaccess/common.htaccess') if engine.COMPRESSION == 'DEFLATE': ht_data += engine.read_local_file(engine.SCRIPT_DIR + '/htaccess/deflate.htaccess') elif engine.COMPRESSION == 'GZIP': ht_data += engine.read_local_file(engine.SCRIPT_DIR + '/htaccess/gzip.htaccess') elif engine.COMPRESSION == 'PRECOMPRESSION': ht_data += engine.read_local_file(engine.SCRIPT_DIR + '/htaccess/precompression.htaccess') elif engine.COMPRESSION == None or engine.COMPRESSION == False or engine.COMPRESSION == 'NONE': pass else: out.log( 'Error: Invalid value for COMPRESSION: ' + str(engine.COMPRESSION) + '. Use DEFLATE, GZIP, PRECOMPRESSION or NONE instead.', 'htaccess', out.LEVEL_ERROR) #caching only for live site if domain == 'live': if engine.CACHING == 'DEVELOPMENT': ht_data += engine.read_local_file( engine.SCRIPT_DIR + '/htaccess/caching-development.htaccess') elif engine.CACHING == 'PRODUCTION': ht_data += engine.read_local_file( engine.SCRIPT_DIR + '/htaccess/caching-production.htaccess') elif engine.CACHING == None or engine.CACHING == False or engine.CACHING == 'NONE': ht_data += engine.read_local_file( engine.SCRIPT_DIR + '/htaccess/caching-none.htaccess') else: out.log( 'Warning: You have not specified a valid browser caching strategy: ' + str(engine.CACHING) + '. Use PRODUCTION, DEVELOPMENT or NONE instead.', 'htaccess', out.LEVEL_WARNING) else: #explicitly disable browsercaching for all assets for local development site ht_data += engine.read_local_file(engine.SCRIPT_DIR + '/htaccess/caching-none.htaccess') if os.path.isfile(engine.LOCAL_WWW_DIR + '/.htaccess.custom'): ht_data += engine.read_local_file(engine.LOCAL_WWW_DIR + '/.htaccess.custom') if engine.IS_WORDPRESS: ht_data += engine.read_local_file(engine.SCRIPT_DIR + '/htaccess/wordpress.htaccess') if domain == 'live': if engine.NEED_BASIC_AUTH: ht_data += engine.read_local_file(engine.SCRIPT_DIR + '/htaccess/basic-auth.htaccess') return ht_data
def save_to_php(filename, data): content = "<?php return " if type(data) == str: content += "'" + data + "';" else: out.log('save_to_php does not support type ' + str(type(data)) + '.', 'compile', out.LEVEL_ERROR) engine.quit() engine.write_local_file(content, filename=filename)
def write_local_file(content, suffix = None, permissions = None, filename = None): if permissions is not None: out.log('Error: Setting permissions in write_local_file is not implemented yet.', 'engine', out.LEVEL_ERROR) if filename is None: filename = get_new_local_file(suffix) if isinstance(content, str): content = unicode(content, 'utf-8' ) file = open(filename, 'w', encoding='utf-8') file.write(content) file.close() return filename
def rename_file(from_file, to_file, file_list): out.log('Registered file for renaming ' + from_file + ' -> ' + to_file, 'engine', out.LEVEL_DEBUG) global current_tmp_file_namespace #define filter function def filter(filename): if filename == from_file: return to_file else: return filename #filter list with that functino file_list[current_tmp_file_namespace] = [filter(filename) for filename in file_list[current_tmp_file_namespace]]
def remove_remote(filename): out.log('remove remote file: ' + filename, 'transfer') if engine.FTP_CLIENT == 'ftp': command = 'delete ' + ftp_path(filename) elif engine.FTP_CLIENT == 'sftp': command = 'rm ' + ftp_path(filename) else: out.log('Unknown ftp client ' + engine.FTP_CLIENT, 'transfer', out.LEVEL_ERROR) engine.quit() ftp.execute(command)
def remote_remote_directory_contents(directory): out.log('remove content of remote directory: ' + dircetory, 'transfer', out.LEVEL_VERBOSE) if engine.FTP_CLIENT == 'ftp': command = 'delete ' + ftp_path(directory) + '/*' elif engine.FTP_CLIENT == 'sftp': command = 'rm ' + ftp_path(directory) + '/*' else: out.log('Unknown ftp client ' + engine.FTP_CLIENT, 'transfer', out.LEVEL_ERROR) engine.quit() ftp.execute(command)
def prepare(overwrite, upload=False): htaccess_file = os.path.abspath(engine.LOCAL_WWW_DIR + '/.htaccess') if os.path.isfile(htaccess_file): if overwrite == 'fail': out.log( '.htaccess already exists. Specify overwrite as parameter or delete the file manually before trying again.', 'htaccess', out.LEVEL_ERROR) engine.quit() if overwrite == 'overwrite': run.local('rm ' + htaccess_file) if upload: transfer.remove_remote('.htaccess')
def remote(command, halt_on_output = False): #tell what happens out.log(command, 'remote', out.LEVEL_VERBOSE) #choose the correct command system if engine.COMMAND_SYSTEM == 'PHP': php.execute(command, halt_on_output) elif engine.COMMAND_SYSTEM == 'SSH': ssh.execute(command) else: #even less implemented out.log("Error Unknown COMMAND_SYSTEM " + engine.COMMAND_SYSTEM, 'remote', out.LEVEL_ERROR) engine.quit()
def remove_remote_multiple(file_list): out.log('removing multiple remote files', 'transfer') command = '' for file in file_list: if engine.FTP_CLIENT == 'ftp': command += 'delete ' + ftp_path(file) + '\n' elif engine.FTP_CLIENT == 'sftp': command += 'rm ' + ftp_path(file) + '\n' else: out.log('Unknown ftp client ' + engine.FTP_CLIENT, 'transfer', out.LEVEL_ERROR) engine.quit() ftp.execute(command)
def create_remote_md5_table(): #register new remote file remote_md5_table_file = engine.get_new_remote_file('json') #run script to create md5 hashes on the remote out.log('calculating remote md5 hashes', 'sync') run.remote_python_script(engine.SCRIPT_DIR + '/py/scripts/create_md5_table.py', engine.NORM_WWW_DIR + ' ' + remote_md5_table_file) #download table file out.log('download remote hash table', 'sync', out.LEVEL_VERBOSE) md5_table_file = transfer.get(remote_md5_table_file) #load it md5_table = load_md5_table(md5_table_file) #and return it return md5_table
def optimize_jpg(quality = 100): out.log('optimizing jpgs...', 'image') for root, dirnames, filenames in os.walk(engine.LOCAL_WWW_DIR): for filename in fnmatch.filter(filenames, '*.jpg'): if os.path.getsize(root + '/' + filename) > 20000: progressive = '--all-progressive' else: progressive = '' if quality < 100: quality = '-m' + str(quality) else: quality = '' run.local('jpegoptim ' + progressive + ' ' + quality + ' "' + root + '/' + filename + '"', ignore_exit_code = True)
def remove_command_file(): global command_file_ready if not command_file_ready: return out.log('removing command file', 'php', out.LEVEL_VERBOSE) try: engine.NORM_COMMAND_FILE except: out.log("Error: Could not find NORM_COMMAND_FILE. No file to remove", 'php', out.LEVEL_WARNING) return transfer.remove_remote(engine.NORM_COMMAND_FILE) command_file_ready = False
def po(): out.log('looking for .po files recursively...', 'compile', out.LEVEL_VERBOSE) compiled_files = 0 files = [] for root, dirnames, filenames in os.walk(engine.LOCAL_WWW_DIR): for filename in fnmatch.filter(filenames, '*.po'): files.append(os.path.join(root, filename)) for po in files: mo = po[:-3] + '.mo' # needs to be refreshed if # 1. there is no .mo file # 2. the .mo file is out of date # 3. the .mo file is not placed in a folder named 'orig' if (not os.path.isfile(mo) or os.path.getmtime(po) > os.path.getmtime(mo)) and ( not os.path.split(os.path.dirname(po))[1] == 'orig'): out.log('compiling ' + po, 'compile') run.local('msgfmt -o ' + mo + ' ' + po) compiled_files += 1 else: out.log('skipping ' + po, 'compile', out.LEVEL_VERBOSE) out.log('all .mo files up to date.', 'compile')
def execute(command): out.log(command, 'ftp', out.LEVEL_VERBOSE) #write the command into file ftp_file = engine.write_local_file(command, 'ftp') #run the ftp file if engine.FTP_CLIENT == 'ftp': ftp_command_line = 'ftp -i ' + engine.FTP_PROTOCOL + '://' + engine.escape( engine.FTP_USER) + ':' + engine.escape( engine.FTP_PASSWORD) + '@' + engine.FTP_HOST + ' <' + ftp_file elif engine.FTP_CLIENT == 'sftp': ftp_command_line = 'sftp -b ' + ftp_file + ' ' + engine.escape( engine.FTP_USER) + '@' + engine.FTP_HOST run.local(ftp_command_line, retry=3)
def get_database_file_description(domain = None): if domain == 'local': description = 'dump-localhost' elif domain == 'remote': try: description = 'dump-' + LIVE_DOMAIN except NameError: description = 'dump-remote-unknown' elif domain is None: description = 'dump' else: out.log('Unknown domain: ' + str(domain), 'engine', out.LEVEL_ERROR) quit() return description
def append_to_hosts(): try: engine.LOCAL_DOMAIN except: out.log( 'Your LOCAL_DOMAIN variable has not been set. Please specify a valid hostname and try again.' ) return out.log( 'appending ' + engine.LOCAL_DOMAIN + ' to ' + engine.LOCAL_ETC_HOSTS, 'apache') run.local('echo "127.0.0.1 ' + engine.LOCAL_DOMAIN + ' # appended by fabric deploy script." >>' + engine.LOCAL_ETC_HOSTS, sudo=True)
def writeDjangoUrlConfig(filenames, urlConfig_destination, urlConfig_name="apiUrls.py", templatePrefix='api/'): """ Creates a Django-style URL Configuration file for all the filenames specified. @input filenames - the file names to write into the urlConfig file @input urlConfig_destination - the destination for the apiUrls file @input urlConfig_name - urlconfig file name (default: 'apiUrls.py') @input templatePrefix - In the render_to_response call, should we look in a subdirectory of the templates dir? (default: 'api/') @return - none """ os.chdir(urlConfig_destination) out.debugPrint("Changed working directory to " + os.getcwd()) f = open(urlConfig_name,'w') # open the file for non-append writing out.log("Opened file for writing:\n\t" + os.path.abspath(f.name)) f.write(out.generateUrlPatterns(filenames, templatePrefix)) # write the urlpatterns to the file out.log("Wrote apiUrls.py file.") # finally, close the file f.close()
def setUpDjango(djangoProjectPath, djangoSettingsModule="settings"): """ Sets up the django environment for a given project to help with the importing of django-related modules. @input djangoProjectPath - the full directory path to the django project we'll be using. @input djangoSettingsModule - the name of the python module to use as the settings for this project. (default = "settings") """ # generate the full module name for the settings module settingsModuleName = os.path.basename(djangoProjectPath) + "." + djangoSettingsModule try: # do all the important settings stuff sys.path.append(os.path.abspath(djangoProjectPath+'/..')) os.environ['DJANGO_SETTINGS_MODULE'] = settingsModuleName from django.core.management import setup_environ settingsModule = importer.importModule(settingsModuleName) out.log("Configured PATH, loaded Django settings.") setup_environ(settingsModule) out.log("Set up Django Environment for project " + os.path.basename(djangoProjectPath)+".") except Exception as e: out.log("Could not set up Django environment - " + str(e), err=True) out.log("Is the Django settings module really called " + settingsModuleName + " ?") out.log("Proceeding anyway. Many of the module imports may fail.", err=True)
def documentModules(moduleNames, exclude=[], destination=".", Path=None): """ Generates pydoc documentation for each module name given and outputs the HTML files into the destination directory. @input moduleNames - a list of names for the modules to document. @input exclude - a list of module and package names that should NOT be documented @input destination - a string indicating the directory path where the documentation HTML should be written. Defaults to "." @input Path - any specific PATH to use? @return - a list of the filenames of all html files which were written. """ # update the path variable with any special info sys.path.append(Path) writtenFiles = [] # list for all files that have been written # change to the appropriate directory os.chdir(destination) # loop through all the module names we were given for modName in moduleNames: # filter out any excluded modules for x in exclude: if modName.find(x) != -1: out.log("Skipping module " + modName) modName = "" # filter out bogus module names if modName == "": continue # import the module and write out the documentation for it. try: M = importModule(modName, Path=Path) out.log("",nl=False) pydoc.writedoc(M) writtenFiles.append(modName+".html") except ImportError as e: # print error msg and proceed to next object out.log("Could not import module " + modName + " - " + str(e), err=True) continue return writtenFiles