def get_git_credentials(self): git_username = None encrypted_password = None if not os.name == 'nt': # Python 2 # git_username = raw_input('Username: '******'Username: '******'None', 'none', 'N', 'n', '--none']: git_username = "" # log_info = "Git username disabled" else: log_info = 'Git username set to ' + git_username # self.update_config_file('git_username', git_username, self.conf_parser, self.config_file_name, log_info) if git_password in ['None', 'none', 'N', 'n', '--none']: git_password = "" # log_info = "Git password disabled" else: log_info = 'Git password set' # self.update_config_file('git_password', self.git_auto.encrypt(git_password), self.conf_parser, self.config_file_name, log_info) # else: # error("Only SSH Key access is enabled on Windows") # git_username = "" # git_password = "" if not encrypted_password == None: git_auto = Git_Auto(self.path) encrypted_password = git_auto.encrypt(git_password) return git_username, encrypted_password
def __init__(self, path, watch=False, timeout=60): self.host = '' self.access_token = '' self.project_id = '' self.project_name = '' self.path = path self.community_id = '' self.workflow_id = '' # default workflow id; MT phase only self.locale = '' self.clone_option = 'on' self.finalized_file = 'off' self.unzip_file = 'on' self.auto_format_option = '' self.download_option = 'clone' self.download_dir = None # directory where downloaded translation will be stored self.watch_locales = set() # if specified, add these target locales to any files in the watch folder self.git_autocommit = None self.git_username = '' self.git_password = '' self.append_option = 'none' self.locale_folders = {} self.default_metadata = {} self.metadata_prompt = False self.metadata_fields = METADATA_FIELDS if not self._is_initialized(): raise exceptions.UninitializedError("This project is not initialized. Please run init command.") self._initialize_self() self.watch = watch self.doc_manager = DocumentManager(self.path) self.folder_manager = FolderManager(self.path) self.timeout = timeout self.api = ApiCalls(self.host, self.access_token, self.watch, self.timeout) self.git_auto = Git_Auto(self.path) self.error_file_name = os.path.join(self.path, CONF_DIR, ERROR_FN)
def __init__(self, path=None, timeout=60): Action.__init__(self, path, True, timeout) self.observers = [] # watchdog observers that will watch the files self.handler = WatchHandler() self.handler.on_modified = self._on_modified self.handler.on_created = self._on_created self.handler.on_moved = self._on_moved self.watch_queue = [] # not much slower than deque unless expecting 100+ items self.locale_delimiter = None self.ignore_ext = [] # file types to ignore as specified by the user self.detected_locales = {} # dict to keep track of detected locales self.watch_folder = True self.timeout = timeout self.updated = {} self.git_auto = Git_Auto(path) self.polled_list = set([]) self.force_poll = False
def __init__(self, path=None, timeout=60): Action.__init__(self, path, True, timeout) self.observers = [] # watchdog observers that will watch the files self.handler = WatchHandler() self.handler.on_modified = self._on_modified self.handler.on_created = self._on_created self.handler.on_moved = self._on_moved self.watch_queue = [ ] # not much slower than deque unless expecting 100+ items self.locale_delimiter = None self.ignore_ext = [] # file types to ignore as specified by the user self.detected_locales = {} # dict to keep track of detected locales self.watch_folder = True self.timeout = timeout self.updated = {} self.git_auto = Git_Auto(path) self.polled_list = set([]) self.force_poll = False self.add = add_action.AddAction(path) self.download = download_action.DownloadAction(path) self.root_path = path self.download_file_paths = set() # set of download file path names
class WatchAction(Action): # def __init__(self, path, remote=False): def __init__(self, path=None, timeout=60): Action.__init__(self, path, True, timeout) self.observers = [] # watchdog observers that will watch the files self.handler = WatchHandler() self.handler.on_modified = self._on_modified self.handler.on_created = self._on_created self.handler.on_moved = self._on_moved self.watch_queue = [] # not much slower than deque unless expecting 100+ items self.locale_delimiter = None self.ignore_ext = [] # file types to ignore as specified by the user self.detected_locales = {} # dict to keep track of detected locales self.watch_folder = True self.timeout = timeout self.updated = {} self.git_auto = Git_Auto(path) self.polled_list = set([]) self.force_poll = False # if remote: # poll lingotek cloud periodically if this option enabled # self.remote_thread = threading.Thread(target=self.poll_remote(), args=()) # self.remote_thread.daemon = True # self.remote_thread.start() def is_translation(self, file_name): locales = locale_list if any('.'+locale in file_name for locale in locales): locales = {v:k for k,v in enumerate(locales) if v in file_name}.keys() replace_target = None for locale in locales: original = file_name file_name = file_name.replace('.'+locale, '') if file_name != original: replace_target = locale break file_name = re.sub('\.{2,}', '.', file_name) file_name = file_name.rstrip('.') doc = self.doc_manager.get_doc_by_prop('file_name', file_name.replace(self.path, '')) if doc: if 'locales' in doc and replace_target in doc['locales']: return True return False def check_remote_doc_exist(self, fn, document_id=None): """ check if a document exists remotely """ if not document_id: entry = self.doc_manager.get_doc_by_prop('file_name', fn) document_id = entry['id'] response = self.api.get_document(document_id) if response.status_code != 200: return False return True def _on_modified(self, event): """ Notify Lingotek cloud when a previously added file is modified """ try: db_entries = self.doc_manager.get_all_entries() in_db = False fn = '' # print event.src_path for entry in db_entries: # print entry['file_name'] if event.src_path.endswith(entry['file_name']): fn = entry['file_name'] in_db = True if not event.is_directory and in_db: try: # check that document is added in TMS before updating if self.check_remote_doc_exist(fn) and self.doc_manager.is_doc_modified(fn, self.path): # logger.info('Detected local content modified: {0}'.format(fn)) # self.update_document_action(os.path.join(self.path, fn)) # logger.info('Updating remote content: {0}'.format(fn)) self.polled_list.remove(fn) self.update_content(fn) except KeyboardInterrupt: for observer in self.observers: observer.stop() except ConnectionError: print("Could not connect to remote server.") restart() except ValueError: print(sys.exc_info()[1]) restart() except KeyboardInterrupt: for observer in self.observers: observer.stop() except Exception as err: restart("Error on modified: "+str(err)+"\nRestarting watch.") def _on_created(self, event): # get path # add action try: file_path = event.src_path # if it's a hidden document, don't do anything if not is_hidden_file(file_path) and not self.is_translation(file_path): relative_path = file_path.replace(self.path, '') title = os.path.basename(os.path.normpath(file_path)) curr_ext = os.path.splitext(file_path)[1] # return if the extension should be ignored or if the path is not a file if curr_ext in self.ignore_ext or not os.path.isfile(file_path): # logger.info("Detected a file with an extension in the ignore list, ignoring..") return # only add or update the document if it's not a hidden document and it's a new file try: if self.doc_manager.is_doc_new(relative_path) and self.watch_folder: self.add_document(file_path, title, locale=self.locale) elif self.doc_manager.is_doc_modified(relative_path, self.path): self.update_content(relative_path) else: return except KeyboardInterrupt: for observer in self.observers: observer.stop() except ConnectionError: print("Could not connect to remote server.") restart() except ValueError: print(sys.exc_info()[1]) restart() doc = self.doc_manager.get_doc_by_prop('file_name', relative_path) if doc: document_id = doc['id'] else: return if self.locale_delimiter: try: # curr_locale = title.split(self.locale_delimiter)[1] # todo locale detection needs to be more robust curr_locale = title.split(self.locale_delimiter)[-2] fixed_locale = map_locale(curr_locale) if fixed_locale: print ("fixed locale: ", fixed_locale) # self.watch_locales.add(fixed_locale) self.detected_locales[document_id] = fixed_locale else: logger.warning('This document\'s detected locale: {0} is not supported.'.format(curr_locale)) except IndexError: logger.warning('Cannot detect locales from file: {0}, not adding any locales'.format(title)) self.watch_add_target(relative_path, document_id) # logger.info('Added new document {0}'.format(title # else: # print("Skipping hidden file "+file_path) except KeyboardInterrupt: for observer in self.observers: observer.stop() # except Exception as err: # restart("Error on created: "+str(err)+"\nRestarting watch.") def _on_moved(self, event): """Used for programs, such as gedit, that modify documents by moving (overwriting) the previous document with the temporary file. Only the moved event contains the name of the destination file.""" try: event = FileSystemEvent(event.dest_path) self._on_modified(event) except KeyboardInterrupt: for observer in self.observers: observer.stop() except Exception as err: restart("Error on moved: "+str(err)+"\nRestarting watch.") def get_watch_locales(self, document_id): """ determine the locales that should be added for a watched doc """ locales = [] if self.detected_locales: try: locales = [self.detected_locales[document_id]] except KeyError: logger.error("Something went wrong. Could not detect a locale") return locales entry = self.doc_manager.get_doc_by_prop("id", document_id) try: locales = [locale for locale in self.watch_locales if locale not in entry['locales']] except KeyError: locales = self.watch_locales return locales def watch_add_target(self, file_name, document_id): if not file_name: title=self.doc_manager.get_doc_by_prop("id", document_id) else: title = os.path.basename(file_name) if document_id not in self.watch_queue: self.watch_queue.append(document_id) # Only add target if doc exists on the cloud if self.check_remote_doc_exist(title, document_id): locales_to_add = self.get_watch_locales(document_id) if locales_to_add == ['[]']: locales_to_add = [] # if len(locales_to_add) == 1: # printStr = "Adding target "+locales_to_add[0] # else: # printStr = "Adding targets " # for target in locales_to_add: # printStr += target+"," # print(printStr) if self.api.get_document(document_id): if self.target_action(title, file_name, locales_to_add, None, None, None, document_id, True) and document_id in self.watch_queue: self.watch_queue.remove(document_id) def process_queue(self): """do stuff with documents in queue (currently just add targets)""" # todo may want to process more than 1 item every "poll".. # if self.watch_queue: # self.watch_add_target(None, self.watch_queue.pop(0)) for document_id in self.watch_queue: self.watch_add_target(None, document_id) def update_content(self, relative_path): if self.update_document_action(os.path.join(self.path, relative_path)): self.updated[relative_path] = 0 logger.info('Updating remote content: {0}'.format(relative_path)) def check_modified(self, doc): # Checks if the version of a document on Lingotek's system is more recent than the local version old_date = doc['last_mod'] response = self.api.get_document(doc['id']) if response.status_code == 200: new_date = response.json()['properties']['modified_date'] # orig_count = response.json()['entities'][1]['properties']['count']['character'] else: print("Document not found on Lingotek Cloud: "+str(doc['name'])) return False if int(old_date)<int(str(new_date)[0:10]): return True return False @retry(logger) def poll_remote(self): """ poll lingotek servers to check if translation is finished """ documents = self.doc_manager.get_all_entries() # todo this gets all documents, not necessarily only ones in watch folder documents_downloaded = False git_commit_message = "" for doc in documents: doc_id = doc['id'] if doc_id in self.watch_queue: # if doc id in queue, not imported yet continue file_name = doc['file_name'] # Wait for Lingotek's system to no longer show translationas as completed if file_name in self.updated: if self.updated[file_name] > 3: self.updated.pop(file_name, None) else: self.updated[file_name] += self.timeout continue try: downloaded = doc['downloaded'] except KeyError: downloaded = [] self.doc_manager.update_document('downloaded', downloaded, doc_id) if file_name not in self.polled_list or self.force_poll: locale_progress = self.import_locale_info(doc_id, True) # Python 2 # for locale, progress in locale_progress.iteritems(): # End Python 2 # Python 3 for locale in locale_progress: progress = locale_progress[locale] # End Python 3 if progress == 100 and locale not in downloaded: document_added = False if (doc['name']+": ") not in git_commit_message: if documents_downloaded: git_commit_message += '; ' git_commit_message += doc['name'] + ": " document_added = True if document_added: git_commit_message += locale else: git_commit_message += ', ' + locale documents_downloaded = True logger.info('Translation completed ({0} - {1})'.format(doc_id, locale)) if self.locale_delimiter: self.download_action(doc_id, locale, False, False) else: self.download_action(doc_id, locale, False) elif progress != 100 and locale in downloaded: # print("Locale "+str(locale)+" for document "+doc['name']+" is no longer completed.") self.doc_manager.remove_element_in_prop(doc_id, 'downloaded', locale) if set(locale_progress.keys()) == set(downloaded): if all(value == 100 for value in locale_progress.values()): self.polled_list.add(file_name) config_file_name, conf_parser = self.init_config_file() git_autocommit = conf_parser.get('main', 'git_autocommit') if git_autocommit == "True" and documents_downloaded == True: username = conf_parser.get('main', 'git_username') password = conf_parser.get('main', 'git_password') self.git_auto.commit(git_commit_message) if username and username != "": if password and password != "": self.git_auto.push(username=username, password=password) else: self.git_auto.push(username=username) else: if password and password != "": self.git_auto.push(password=password) else: self.git_auto.push() def complete_path(self, file_location): # print("self.path: "+self.path) # print("file_location: "+file_location) # print("abs file_location: "+os.path.abspath(file_location)) abspath=os.path.abspath(file_location) # norm_path = os.path.abspath(file_location).replace(self.path, '') print return abspath.rstrip(os.sep) def watch_action(self, ignore, delimiter=None, no_folders=False, force_poll=False): # watch_paths, ignore, delimiter=None, no_folders=False): # print self.path watch_paths = None if not watch_paths: watch_paths = self.folder_manager.get_file_names() print("Watch paths: "+str(watch_paths)) for i in range(len(watch_paths)): watch_paths[i] = get_relative_path(self.path, watch_paths[i]) else: watch_paths_list = [] for path in watch_paths: watch_paths_list.append(path.rstrip(os.sep)) watch_paths = watch_paths_list if len(watch_paths) and not no_folders: # Use watch path specified as an option/parameter self.watch_folders = True else: self.watch_folder = False # Only watch added files if self.watch_folder: watch_message = "Watching for updates in " for i in range(len(watch_paths)): watch_paths[i] = self.complete_path(watch_paths[i]) watch_message += "{0}".format(watch_paths[i]) if i < len(watch_paths)-1: watch_message += " " print (watch_message) else: print ("Watching for updates to added documents") if force_poll: self.force_poll = True self.ignore_ext.extend(ignore) self.locale_delimiter = delimiter for watch_path in watch_paths: observer = Observer() observer.schedule(self.handler, path=watch_path, recursive=True) observer.start() self.observers.append(observer) queue_timeout = 3 # start_time = time.clock() try: while True: self.poll_remote() current_timeout = self.timeout while len(self.watch_queue) and current_timeout > 0: self.process_queue() time.sleep(queue_timeout) current_timeout -= queue_timeout time.sleep(self.timeout) except KeyboardInterrupt: for observer in self.observers: observer.stop() # except Exception as err: # restart("Error: "+str(err)+"\nRestarting watch.") for observer in self.observers: observer.join()
def __init__(self, path, download): Action.__init__(self, path) self.git_auto = Git_Auto(path) self.download = download