Example #1
0
def get_valid_locales(api, entered_locales, operation_text):
    """Return the list of valid locales, checking locales either remotely or using a local list of locales."""
    valid_locales = []
    response = api.list_locales()
    remote_check = False
    if response.status_code == 200:
        remote_check = True
    locale_json = response.json()
    for entry in locale_json:
        valid_locales.append(locale_json[entry]['locale'])
    locales = []
    if (len(entered_locales) == 0
            or (len(entered_locales) == 1 and entered_locales[0] == "[]")):
        logger.warning(
            'No locales have been assigned to this document.  Please add them using \'ltk request\'.'
        )
    else:
        for locale in entered_locales:
            check_locale = locale.replace("-", "_")
            if remote_check and check_locale not in valid_locales or not remote_check and not check_locale in locale_list:
                logger.warning(
                    'The locale code "' + str(locale) + '" failed to be ' +
                    operation_text +
                    ' since it is invalid (see "ltk list -l" for the list of valid codes).'
                )
            else:
                locales.append(locale)
    return locales
 def rm_document(self, file_name, useID, force, doc_name=None):
     doc = None
     if not useID:
         relative_path = file_name.replace(self.path, '')
         doc = self.doc_manager.get_doc_by_prop('file_name', relative_path)
         title = os.path.basename(os.path.normpath(file_name))
         # print("relative_path: "+relative_path)
         try:
             doc = self.doc_manager.get_doc_by_prop('file_name', relative_path)
             document_id = doc['id']
         except TypeError: # Documents specified by name must be found in the local database to be removed.
             logger.warning("Document name specified for remove doesn't exist locally: {0}".format(relative_path))
             return
             # raise exceptions.ResourceNotFound("Document name specified doesn't exist: {0}".format(document_name))
     else:
         document_id = file_name
         doc = self.doc_manager.get_doc_by_prop('id', document_id)
         # raise exceptions.ResourceNotFound("Document name specified doesn't exist: {0}".format(document_name))
         if doc:
             file_name = doc['file_name']
     response = self.api.document_delete(document_id)
     #print (response)
     if response.status_code != 204:            
         # raise_error(response.json(), "Failed to delete document {0}".format(document_name), True)
         logger.error("Failed to delete document {0} remotely".format(file_name))
     else:
         if doc_name:
             logger.info("{0} ({1}) has been deleted remotely.".format(doc_name, file_name))
         else:
             logger.info("{0} has been deleted remotely.".format(file_name))
         if doc:
             if force:               
                 self.delete_local(file_name, document_id)
             self.doc_manager.remove_element(document_id)
Example #3
0
def get_valid_locales(api, entered_locales):
    """Return the list of valid locales, checking locales either remotely or using a local list of locales."""
    valid_locales = []
    response = api.list_locales()
    remote_check = False
    if response.status_code == 200:
        remote_check = True
    locale_json = response.json()
    for entry in locale_json:
        valid_locales.append(locale_json[entry]["locale"])
    locales = []
    if len(entered_locales) == 0 or (len(entered_locales) == 1 and entered_locales[0] == "[]"):
        logger.warning("No locales have been assigned to this document.  Please add them using 'ltk request'.")
    else:
        for locale in entered_locales:
            locale = locale.replace("-", "_")
            if remote_check and locale not in valid_locales or not remote_check and not locale in locale_list:
                logger.warning(
                    'The locale code "'
                    + str(locale)
                    + '" failed to be added since it is invalid (see "ltk list -l" for the list of valid codes).'
                )
            else:
                locales.append(locale)
    return locales
Example #4
0
    def add_document(self, file_name, title, doc_metadata={}, **kwargs):
        ''' adds the document to Lingotek cloud and the db '''
        if ltk.check_connection.check_for_connection() == False:
            logger.warning(
                "Cannot connect to network. Documents added to the watch folder will be translated after you reconnect to the network."
            )
            while ltk.check_connection.check_for_connection() == False:
                time.sleep(15)

        if self.is_hidden_file(file_name):
            return
        try:
            if not 'locale' in kwargs or not kwargs['locale']:
                locale = self.locale
            else:
                locale = kwargs['locale']

            # add document to Lingotek cloud
            response = self.api.add_document(
                locale, file_name, self.project_id,
                self.append_location(title, file_name), doc_metadata, **kwargs)
            if response.status_code == 402:
                raise_error(response.json(), "", True)
            elif response.status_code != 202:
                raise_error(response.json(),
                            "Failed to add document {0}\n".format(title), True)
            else:
                title = self.append_location(title, file_name)
                logger.info('Added document {0} with ID {1}\n'.format(
                    title,
                    response.json()['properties']['id']))
                relative_path = self.norm_path(file_name)

                # add document to the db
                if 'download_folder' in kwargs and kwargs['download_folder']:
                    self._add_document(
                        relative_path, title,
                        response.json()['properties']['id'],
                        response.json()['properties']['process_id'],
                        kwargs['download_folder'])
                else:
                    self._add_document(
                        relative_path, title,
                        response.json()['properties']['id'],
                        response.json()['properties']['process_id'])
                if 'translation_locale_code' in kwargs and kwargs[
                        'translation_locale_code']:
                    self._update_document(relative_path, None,
                                          kwargs['translation_locale_code'])
        except KeyboardInterrupt:
            raise_error("", "Canceled adding document\n")
        except Exception as e:
            log_error(self.error_file_name, e)
            if 'string indices must be integers' in str(
                    e) or 'Expecting value: line 1 column 1' in str(e):
                logger.error("Error connecting to Lingotek's TMS\n")
            else:
                logger.error("Error on adding document \n" + str(file_name) +
                             ": " + str(e))
Example #5
0
def check_response(response):
    try:
        if response and response.text:
            if response.json():
                return True
    except ValueError:
        logger.warning("Could not connect to Lingotek")
        return
Example #6
0
def check_response(response):
    try:
        if response and response.text:
            if response.json():
                return True
    except ValueError:
        logger.warning("Could not connect to Lingotek")
        return
Example #7
0
 def _on_created(self, event):
     # get path
     # add action
     try:
         file_path = event.src_path
         # if it's a hidden document, don't do anything
         if not is_hidden_file(file_path) and not self.is_translation(file_path):
             relative_path = file_path.replace(self.path, '')
             title = os.path.basename(os.path.normpath(file_path))
             curr_ext = os.path.splitext(file_path)[1]
             # return if the extension should be ignored or if the path is not a file
             if curr_ext in self.ignore_ext or not os.path.isfile(file_path):
                 # logger.info("Detected a file with an extension in the ignore list, ignoring..")
                 return
             # only add or update the document if it's not a hidden document and it's a new file
             try:
                 if self.doc_manager.is_doc_new(relative_path) and self.watch_folder:
                     self.add_document(file_path, title, locale=self.locale)
                 elif self.doc_manager.is_doc_modified(relative_path, self.path):
                     self.update_content(relative_path)
                 else:
                     return
             except KeyboardInterrupt:
                 for observer in self.observers:
                     observer.stop()
             except ConnectionError:
                 print("Could not connect to remote server.")
                 restart()
             except ValueError:
                 print(sys.exc_info()[1])
                 restart()
             doc = self.doc_manager.get_doc_by_prop('file_name', relative_path)
             if doc:
                 document_id = doc['id']
             else:
                 return
             if self.locale_delimiter:
                 try:
                     # curr_locale = title.split(self.locale_delimiter)[1]
                     # todo locale detection needs to be more robust
                     curr_locale = title.split(self.locale_delimiter)[-2]
                     fixed_locale = map_locale(curr_locale)
                     if fixed_locale:
                         print ("fixed locale: ", fixed_locale)
                         # self.watch_locales.add(fixed_locale)
                         self.detected_locales[document_id] = fixed_locale
                     else:
                         logger.warning('This document\'s detected locale: {0} is not supported.'.format(curr_locale))
                 except IndexError:
                     logger.warning('Cannot detect locales from file: {0}, not adding any locales'.format(title))
             self.watch_add_target(relative_path, document_id)
             # logger.info('Added new document {0}'.format(title
         # else:
         #     print("Skipping hidden file "+file_path)
     except KeyboardInterrupt:
         for observer in self.observers:
             observer.stop()
    def clean_action(self, force, dis_all, document_name):
        if dis_all:
            # disassociate everything
            self.doc_manager.clear_all()
            return

        if document_name:
            try:
                entry = self.doc_manager.get_doc_by_prop('name', document_name)
                document_id = entry['id']
                self.doc_manager.remove_element(document_id)
            except TypeError:
                logger.warning("Document name specified for clean doesn't exist: {0}".format(document_name))
            return

        response = self.api.list_documents(self.project_id)
        local_ids = self.doc_manager.get_doc_ids()
        tms_doc_ids = []
        if response.status_code == 200:
            tms_documents = response.json()['entities']
            for entity in tms_documents:
                tms_doc_ids.append(entity['properties']['id'])
        elif response.status_code == 204:
            pass
        else:
            raise_error(response.json(), 'Error trying to list documents in TMS for cleaning')
        locals_to_delete = [x for x in local_ids if x not in tms_doc_ids]

        # check local files
        db_entries = self.doc_manager.get_all_entries()
        for entry in db_entries:
            # if local file doesn't exist, remove entry
            if not os.path.isfile(os.path.join(self.path, entry['file_name'])):
                locals_to_delete.append(entry['id'])

        # remove entry for local doc -- possibly delete local file too?
        if locals_to_delete:
            for curr_id in locals_to_delete:
                removed_doc = self.doc_manager.get_doc_by_prop('id', curr_id)
                if not removed_doc:
                    continue
                removed_title = removed_doc['name']
                # todo somehow this line^ doc is null... after delete files remotely, then delete locally
                if force:
                    self.delete_local(removed_title, curr_id)
                self.doc_manager.remove_element(curr_id)
                logger.info('Removing association for document {0}'.format(removed_title))
        else:
            logger.info('Local documents already up-to-date with Lingotek cloud')
            return
        logger.info('Cleaned up associations between local documents and Lingotek cloud')
Example #9
0
 def append_location(self, name, path_to_file, in_directory=False):
     repo_directory = path_to_file
     path_sep = os.sep
     config_file_name, conf_parser = self.init_config_file()
     if not conf_parser.has_option('main', 'append_option'): self.update_config_file('append_option', 'none', conf_parser, config_file_name, 'Update: Added optional file location appending (ltk config --help)')
     append_option = conf_parser.get('main', 'append_option')
     if not in_directory:
         while repo_directory and repo_directory != "" and not (os.path.isdir(repo_directory + os.sep+".ltk")):
             repo_directory = repo_directory.split(path_sep)[:-1]
             repo_directory = path_sep.join(repo_directory)
         if repo_directory == "" and append_option != 'none':
             logger.warning('Error: File must be contained within an ltk-initialized directory')
             return name
         path_to_file = path_to_file.replace(repo_directory, '', 1).strip(os.sep)
     if append_option == 'none': return name
     elif append_option == 'full': return '{0} ({1})'.format(name, path_to_file.rstrip(name).rstrip(os.sep))
     elif len(append_option) > 5 and append_option[:5] == 'name:':
         folder_name = append_option[5:]
         if folder_name in path_to_file:
             return '{0} ({1})'.format(name, path_to_file[path_to_file.find(folder_name)+len(folder_name):].rstrip(name).strip(os.sep))
         else: return '{0} ({1})'.format(name, path_to_file.rstrip(name).rstrip(os.sep))
     elif len(append_option) > 7 and append_option[:7] == 'number:':
         try: folder_number = int(append_option[7:])
         except ValueError:
             logger.warning('Error: Value after "number" must be an integer')
             return name
         if(folder_number >=0):
             return '{0} ({1})'.format(name, path_sep.join(path_to_file.rstrip(name).rstrip(os.sep).split(path_sep)[(-1*folder_number) if folder_number != 0 else len(path_to_file):]))
         else:
             logger.warning('Error: Value after "number" must be a non-negative integer')
             return name
     else:
         logger.warning('Error: Invalid value listed for append option. Please update; see ltk config --help')
Example #10
0
 def validate_metadata_fields(self, field_options):
     if field_options.lower() == 'all' or field_options == '':
         return True, METADATA_FIELDS
     else:
         converted = field_options.replace(
             ", ", ","
         )  #allows for a comma-separated list with or without a single space after commas
         options = converted.split(",")
         for option in options:
             if option not in METADATA_FIELDS:
                 logger.warning(
                     "Error: {0} is not a valid metadata field".format(
                         option))
                 return False, None
         return True, options
def reinit(host, project_path, delete, reset):
    if is_initialized(project_path) and not reset:
        logger.warning('This project is already initialized!')
        if not delete:
            return False
        confirm = 'not confirmed'
        while confirm != 'y' and confirm != 'Y' and confirm != 'N' and confirm != 'n' and confirm != '':
            confirm = input(
                "Are you sure you want to delete the current project? "
                "This will also delete the project in your community. [y/N]: ")
        # confirm if would like to delete existing folder
        if not confirm or confirm in ['n', 'N']:
            return False
        else:
            # delete the corresponding project online
            logger.info('Deleting old project folder and creating new one...')
            config_file_name = os.path.join(project_path, CONF_DIR, CONF_FN)
            if os.path.isfile(config_file_name):
                old_config = configparser.ConfigParser()
                old_config.read(config_file_name)
                project_id = old_config.get('main', 'project_id')
                access_token = old_config.get('main', 'access_token')
                api = ApiCalls(host, access_token)
                response = api.delete_project(project_id)
                if response.status_code != 204 and response.status_code != 404:
                    try:
                        error = response.json()['messages'][0]
                        raise exceptions.RequestFailedError(error)
                    except (AttributeError, IndexError):
                        raise exceptions.RequestFailedError("Failed to delete and re-initialize project")
                # delete existing folder
                to_remove = os.path.join(project_path, CONF_DIR)
                shutil.rmtree(to_remove)
            else:
                raise exceptions.ResourceNotFound("Cannot find config file, please re-initialize project")
            return access_token
    return True
Example #12
0
 def watch_action(
     self,
     ignore,
     delimiter=None,
     no_folders=False,
     force_poll=False
 ):  # watch_paths, ignore, delimiter=None, no_folders=False):
     # print self.path
     watch_paths = None
     if not watch_paths:
         watch_paths = self.folder_manager.get_file_names()
         for i in range(len(watch_paths)):
             watch_paths[i] = get_relative_path(self.path, watch_paths[i])
     else:
         watch_paths_list = []
         for path in watch_paths:
             watch_paths_list.append(path.rstrip(os.sep))
         watch_paths = watch_paths_list
     if len(watch_paths) and not no_folders:
         self.watch_folder = True
     else:
         watch_paths = [os.getcwd()]
     if self.watch_folder:
         watch_message = "Watching for updates in "
         for i in range(len(watch_paths)):
             watch_paths[i] = self.complete_path(watch_paths[i])
             watch_message += "{0}".format(watch_paths[i])
             if i < len(watch_paths) - 1:
                 watch_message += " "
         print(watch_message)
     else:
         print("Watching for updates to added documents")
     if force_poll:
         self.force_poll = True
     self.ignore_ext.extend(ignore)
     self.locale_delimiter = delimiter
     for watch_path in watch_paths:
         observer = Observer()
         observer.schedule(self.handler, path=watch_path, recursive=True)
         try:
             observer.start()
         except OSError as e:
             logger.warning(
                 "Watching too many items, please be more specific by using ltk add on the files and folders that should be watched"
             )
             return
         self.observers.append(observer)
     queue_timeout = 3
     # start_time = time.clock()
     try:
         while True:
             if ltk.check_connection.check_for_connection():
                 self.poll_remote()
                 current_timeout = self.timeout
                 while len(self.watch_queue) and current_timeout > 0:
                     self.process_queue()
                     time.sleep(queue_timeout)
                     current_timeout -= queue_timeout
             time.sleep(
                 current_timeout)  # default 60 sec total, 3 already taken
     except KeyboardInterrupt:
         for observer in self.observers:
             observer.stop()
     # except Exception as err:
     #     restart("Error: "+str(err)+"\nRestarting watch.")
     for observer in self.observers:
         observer.join()
Example #13
0
    def _on_created(self, event):
        # get path
        # add action
        try:
            db_entries = self.doc_manager.get_all_entries()
            in_db = False
            fn = ''
            for entry in db_entries:
                if event.src_path.endswith(entry['file_name']):
                    fn = entry['file_name']
                    in_db = True
            if not event.is_directory and in_db:
                self._on_modified(event)
            else:
                file_path = event.src_path
                # if created file was a downloaded translation, don't add to poll. Prevents recursion from downloaded translation files when ltk watch is running
                if file_path in self.download_file_paths:
                    self.download_file_paths.remove(file_path)
                    return
                # if it's a hidden document, don't do anything
                if not self.is_hidden_file(
                        file_path) and not self.is_translation(file_path):
                    relative_path = file_path.replace(self.path, '')
                    title = os.path.basename(os.path.normpath(file_path))
                    curr_ext = os.path.splitext(file_path)[1]
                    # return if the extension should be ignored or if the path is not a file
                    if curr_ext in self.ignore_ext or not os.path.isfile(
                            file_path):
                        # logger.info("Detected a file with an extension in the ignore list, ignoring..")
                        return
                    # only add or update the document if it's not a hidden document and it's a new file
                    try:
                        if self.doc_manager.is_doc_new(
                                relative_path,
                                self.root_path) and self.watch_folder:
                            #testing
                            #self.polled_list.add(relative_path) #test that this doesn't break other areas of watch
                            #end testing

                            self.add_document(file_path,
                                              title,
                                              locale=self.locale)

                        elif self.doc_manager.is_doc_modified(
                                relative_path, self.path):
                            self.update_content(relative_path)
                        else:
                            return
                    except KeyboardInterrupt:
                        for observer in self.observers:
                            observer.stop()
                    except ConnectionError:
                        print("Could not connect to remote server.")
                        restart()
                    except ValueError:
                        print(sys.exc_info()[1])
                        restart()
                    doc = self.doc_manager.get_doc_by_prop(
                        'file_name', relative_path)
                    if doc:
                        document_id = doc['id']
                    else:
                        return
                    if self.locale_delimiter:
                        try:
                            # curr_locale = title.split(self.locale_delimiter)[1]
                            # todo locale detection needs to be more robust
                            curr_locale = title.split(
                                self.locale_delimiter)[-2]
                            fixed_locale = map_locale(curr_locale)
                            if fixed_locale:
                                print("fixed locale: ", fixed_locale)
                                # self.watch_locales.add(fixed_locale)
                                self.detected_locales[
                                    document_id] = fixed_locale
                            else:
                                logger.warning(
                                    'This document\'s detected locale: {0} is not supported.'
                                    .format(curr_locale))
                        except IndexError:
                            logger.warning(
                                'Cannot detect locales from file: {0}, not adding any locales'
                                .format(title))
                    self.watch_add_target(relative_path, document_id)
                    # logger.info('Added new document {0}'.format(title
                # else:
                #     print("Skipping hidden file "+file_path)
        except KeyboardInterrupt:
            for observer in self.observers:
                observer.stop()
 def handleError(self):
     if self.watch:
         logger.warning("Could not connect to Lingotek")
         restart("Restarting watch", self.timeout)
     else:
         raise ConnectionFailed("Could not connect to Lingotek")
Example #15
0
 def handleError(self):
     if self.watch:
         logger.warning("Could not connect to Lingotek")
         restart("Restarting watch", self.timeout)
     else:
         raise ConnectionFailed("Could not connect to Lingotek")