def get_valid_locales(api, entered_locales, operation_text):
"""Return the list of valid locales, checking locales either remotely or using a local list of locales."""
valid_locales = []
response = api.list_locales()
remote_check = False
if response.status_code == 200:
remote_check = True
locale_json = response.json()
for entry in locale_json:
valid_locales.append(locale_json[entry]['locale'])
locales = []
if (len(entered_locales) == 0
or (len(entered_locales) == 1 and entered_locales[0] == "[]")):
logger.warning(
'No locales have been assigned to this document. Please add them using \'ltk request\'.'
)
else:
for locale in entered_locales:
check_locale = locale.replace("-", "_")
if remote_check and check_locale not in valid_locales or not remote_check and not check_locale in locale_list:
logger.warning(
'The locale code "' + str(locale) + '" failed to be ' +
operation_text +
' since it is invalid (see "ltk list -l" for the list of valid codes).'
)
else:
locales.append(locale)
return locales
def rm_document(self, file_name, useID, force, doc_name=None):
doc = None
if not useID:
relative_path = file_name.replace(self.path, '')
doc = self.doc_manager.get_doc_by_prop('file_name', relative_path)
title = os.path.basename(os.path.normpath(file_name))
# print("relative_path: "+relative_path)
try:
doc = self.doc_manager.get_doc_by_prop('file_name', relative_path)
document_id = doc['id']
except TypeError: # Documents specified by name must be found in the local database to be removed.
logger.warning("Document name specified for remove doesn't exist locally: {0}".format(relative_path))
return
# raise exceptions.ResourceNotFound("Document name specified doesn't exist: {0}".format(document_name))
else:
document_id = file_name
doc = self.doc_manager.get_doc_by_prop('id', document_id)
# raise exceptions.ResourceNotFound("Document name specified doesn't exist: {0}".format(document_name))
if doc:
file_name = doc['file_name']
response = self.api.document_delete(document_id)
#print (response)
if response.status_code != 204:
# raise_error(response.json(), "Failed to delete document {0}".format(document_name), True)
logger.error("Failed to delete document {0} remotely".format(file_name))
else:
if doc_name:
logger.info("{0} ({1}) has been deleted remotely.".format(doc_name, file_name))
else:
logger.info("{0} has been deleted remotely.".format(file_name))
if doc:
if force:
self.delete_local(file_name, document_id)
self.doc_manager.remove_element(document_id)
def get_valid_locales(api, entered_locales):
"""Return the list of valid locales, checking locales either remotely or using a local list of locales."""
valid_locales = []
response = api.list_locales()
remote_check = False
if response.status_code == 200:
remote_check = True
locale_json = response.json()
for entry in locale_json:
valid_locales.append(locale_json[entry]["locale"])
locales = []
if len(entered_locales) == 0 or (len(entered_locales) == 1 and entered_locales[0] == "[]"):
logger.warning("No locales have been assigned to this document. Please add them using 'ltk request'.")
else:
for locale in entered_locales:
locale = locale.replace("-", "_")
if remote_check and locale not in valid_locales or not remote_check and not locale in locale_list:
logger.warning(
'The locale code "'
+ str(locale)
+ '" failed to be added since it is invalid (see "ltk list -l" for the list of valid codes).'
)
else:
locales.append(locale)
return locales
def add_document(self, file_name, title, doc_metadata={}, **kwargs):
''' adds the document to Lingotek cloud and the db '''
if ltk.check_connection.check_for_connection() == False:
logger.warning(
"Cannot connect to network. Documents added to the watch folder will be translated after you reconnect to the network."
)
while ltk.check_connection.check_for_connection() == False:
time.sleep(15)
if self.is_hidden_file(file_name):
return
try:
if not 'locale' in kwargs or not kwargs['locale']:
locale = self.locale
else:
locale = kwargs['locale']
# add document to Lingotek cloud
response = self.api.add_document(
locale, file_name, self.project_id,
self.append_location(title, file_name), doc_metadata, **kwargs)
if response.status_code == 402:
raise_error(response.json(), "", True)
elif response.status_code != 202:
raise_error(response.json(),
"Failed to add document {0}\n".format(title), True)
else:
title = self.append_location(title, file_name)
logger.info('Added document {0} with ID {1}\n'.format(
title,
response.json()['properties']['id']))
relative_path = self.norm_path(file_name)
# add document to the db
if 'download_folder' in kwargs and kwargs['download_folder']:
self._add_document(
relative_path, title,
response.json()['properties']['id'],
response.json()['properties']['process_id'],
kwargs['download_folder'])
else:
self._add_document(
relative_path, title,
response.json()['properties']['id'],
response.json()['properties']['process_id'])
if 'translation_locale_code' in kwargs and kwargs[
'translation_locale_code']:
self._update_document(relative_path, None,
kwargs['translation_locale_code'])
except KeyboardInterrupt:
raise_error("", "Canceled adding document\n")
except Exception as e:
log_error(self.error_file_name, e)
if 'string indices must be integers' in str(
e) or 'Expecting value: line 1 column 1' in str(e):
logger.error("Error connecting to Lingotek's TMS\n")
else:
logger.error("Error on adding document \n" + str(file_name) +
": " + str(e))
def check_response(response):
try:
if response and response.text:
if response.json():
return True
except ValueError:
logger.warning("Could not connect to Lingotek")
return
def check_response(response):
try:
if response and response.text:
if response.json():
return True
except ValueError:
logger.warning("Could not connect to Lingotek")
return
def _on_created(self, event):
# get path
# add action
try:
file_path = event.src_path
# if it's a hidden document, don't do anything
if not is_hidden_file(file_path) and not self.is_translation(file_path):
relative_path = file_path.replace(self.path, '')
title = os.path.basename(os.path.normpath(file_path))
curr_ext = os.path.splitext(file_path)[1]
# return if the extension should be ignored or if the path is not a file
if curr_ext in self.ignore_ext or not os.path.isfile(file_path):
# logger.info("Detected a file with an extension in the ignore list, ignoring..")
return
# only add or update the document if it's not a hidden document and it's a new file
try:
if self.doc_manager.is_doc_new(relative_path) and self.watch_folder:
self.add_document(file_path, title, locale=self.locale)
elif self.doc_manager.is_doc_modified(relative_path, self.path):
self.update_content(relative_path)
else:
return
except KeyboardInterrupt:
for observer in self.observers:
observer.stop()
except ConnectionError:
print("Could not connect to remote server.")
restart()
except ValueError:
print(sys.exc_info()[1])
restart()
doc = self.doc_manager.get_doc_by_prop('file_name', relative_path)
if doc:
document_id = doc['id']
else:
return
if self.locale_delimiter:
try:
# curr_locale = title.split(self.locale_delimiter)[1]
# todo locale detection needs to be more robust
curr_locale = title.split(self.locale_delimiter)[-2]
fixed_locale = map_locale(curr_locale)
if fixed_locale:
print ("fixed locale: ", fixed_locale)
# self.watch_locales.add(fixed_locale)
self.detected_locales[document_id] = fixed_locale
else:
logger.warning('This document\'s detected locale: {0} is not supported.'.format(curr_locale))
except IndexError:
logger.warning('Cannot detect locales from file: {0}, not adding any locales'.format(title))
self.watch_add_target(relative_path, document_id)
# logger.info('Added new document {0}'.format(title
# else:
# print("Skipping hidden file "+file_path)
except KeyboardInterrupt:
for observer in self.observers:
observer.stop()
def clean_action(self, force, dis_all, document_name):
if dis_all:
# disassociate everything
self.doc_manager.clear_all()
return
if document_name:
try:
entry = self.doc_manager.get_doc_by_prop('name', document_name)
document_id = entry['id']
self.doc_manager.remove_element(document_id)
except TypeError:
logger.warning("Document name specified for clean doesn't exist: {0}".format(document_name))
return
response = self.api.list_documents(self.project_id)
local_ids = self.doc_manager.get_doc_ids()
tms_doc_ids = []
if response.status_code == 200:
tms_documents = response.json()['entities']
for entity in tms_documents:
tms_doc_ids.append(entity['properties']['id'])
elif response.status_code == 204:
pass
else:
raise_error(response.json(), 'Error trying to list documents in TMS for cleaning')
locals_to_delete = [x for x in local_ids if x not in tms_doc_ids]
# check local files
db_entries = self.doc_manager.get_all_entries()
for entry in db_entries:
# if local file doesn't exist, remove entry
if not os.path.isfile(os.path.join(self.path, entry['file_name'])):
locals_to_delete.append(entry['id'])
# remove entry for local doc -- possibly delete local file too?
if locals_to_delete:
for curr_id in locals_to_delete:
removed_doc = self.doc_manager.get_doc_by_prop('id', curr_id)
if not removed_doc:
continue
removed_title = removed_doc['name']
# todo somehow this line^ doc is null... after delete files remotely, then delete locally
if force:
self.delete_local(removed_title, curr_id)
self.doc_manager.remove_element(curr_id)
logger.info('Removing association for document {0}'.format(removed_title))
else:
logger.info('Local documents already up-to-date with Lingotek cloud')
return
logger.info('Cleaned up associations between local documents and Lingotek cloud')
def append_location(self, name, path_to_file, in_directory=False):
repo_directory = path_to_file
path_sep = os.sep
config_file_name, conf_parser = self.init_config_file()
if not conf_parser.has_option('main', 'append_option'): self.update_config_file('append_option', 'none', conf_parser, config_file_name, 'Update: Added optional file location appending (ltk config --help)')
append_option = conf_parser.get('main', 'append_option')
if not in_directory:
while repo_directory and repo_directory != "" and not (os.path.isdir(repo_directory + os.sep+".ltk")):
repo_directory = repo_directory.split(path_sep)[:-1]
repo_directory = path_sep.join(repo_directory)
if repo_directory == "" and append_option != 'none':
logger.warning('Error: File must be contained within an ltk-initialized directory')
return name
path_to_file = path_to_file.replace(repo_directory, '', 1).strip(os.sep)
if append_option == 'none': return name
elif append_option == 'full': return '{0} ({1})'.format(name, path_to_file.rstrip(name).rstrip(os.sep))
elif len(append_option) > 5 and append_option[:5] == 'name:':
folder_name = append_option[5:]
if folder_name in path_to_file:
return '{0} ({1})'.format(name, path_to_file[path_to_file.find(folder_name)+len(folder_name):].rstrip(name).strip(os.sep))
else: return '{0} ({1})'.format(name, path_to_file.rstrip(name).rstrip(os.sep))
elif len(append_option) > 7 and append_option[:7] == 'number:':
try: folder_number = int(append_option[7:])
except ValueError:
logger.warning('Error: Value after "number" must be an integer')
return name
if(folder_number >=0):
return '{0} ({1})'.format(name, path_sep.join(path_to_file.rstrip(name).rstrip(os.sep).split(path_sep)[(-1*folder_number) if folder_number != 0 else len(path_to_file):]))
else:
logger.warning('Error: Value after "number" must be a non-negative integer')
return name
else:
logger.warning('Error: Invalid value listed for append option. Please update; see ltk config --help')
def validate_metadata_fields(self, field_options):
if field_options.lower() == 'all' or field_options == '':
return True, METADATA_FIELDS
else:
converted = field_options.replace(
", ", ","
) #allows for a comma-separated list with or without a single space after commas
options = converted.split(",")
for option in options:
if option not in METADATA_FIELDS:
logger.warning(
"Error: {0} is not a valid metadata field".format(
option))
return False, None
return True, options
def reinit(host, project_path, delete, reset):
if is_initialized(project_path) and not reset:
logger.warning('This project is already initialized!')
if not delete:
return False
confirm = 'not confirmed'
while confirm != 'y' and confirm != 'Y' and confirm != 'N' and confirm != 'n' and confirm != '':
confirm = input(
"Are you sure you want to delete the current project? "
"This will also delete the project in your community. [y/N]: ")
# confirm if would like to delete existing folder
if not confirm or confirm in ['n', 'N']:
return False
else:
# delete the corresponding project online
logger.info('Deleting old project folder and creating new one...')
config_file_name = os.path.join(project_path, CONF_DIR, CONF_FN)
if os.path.isfile(config_file_name):
old_config = configparser.ConfigParser()
old_config.read(config_file_name)
project_id = old_config.get('main', 'project_id')
access_token = old_config.get('main', 'access_token')
api = ApiCalls(host, access_token)
response = api.delete_project(project_id)
if response.status_code != 204 and response.status_code != 404:
try:
error = response.json()['messages'][0]
raise exceptions.RequestFailedError(error)
except (AttributeError, IndexError):
raise exceptions.RequestFailedError("Failed to delete and re-initialize project")
# delete existing folder
to_remove = os.path.join(project_path, CONF_DIR)
shutil.rmtree(to_remove)
else:
raise exceptions.ResourceNotFound("Cannot find config file, please re-initialize project")
return access_token
return True
def watch_action(
self,
ignore,
delimiter=None,
no_folders=False,
force_poll=False
): # watch_paths, ignore, delimiter=None, no_folders=False):
# print self.path
watch_paths = None
if not watch_paths:
watch_paths = self.folder_manager.get_file_names()
for i in range(len(watch_paths)):
watch_paths[i] = get_relative_path(self.path, watch_paths[i])
else:
watch_paths_list = []
for path in watch_paths:
watch_paths_list.append(path.rstrip(os.sep))
watch_paths = watch_paths_list
if len(watch_paths) and not no_folders:
self.watch_folder = True
else:
watch_paths = [os.getcwd()]
if self.watch_folder:
watch_message = "Watching for updates in "
for i in range(len(watch_paths)):
watch_paths[i] = self.complete_path(watch_paths[i])
watch_message += "{0}".format(watch_paths[i])
if i < len(watch_paths) - 1:
watch_message += " "
print(watch_message)
else:
print("Watching for updates to added documents")
if force_poll:
self.force_poll = True
self.ignore_ext.extend(ignore)
self.locale_delimiter = delimiter
for watch_path in watch_paths:
observer = Observer()
observer.schedule(self.handler, path=watch_path, recursive=True)
try:
observer.start()
except OSError as e:
logger.warning(
"Watching too many items, please be more specific by using ltk add on the files and folders that should be watched"
)
return
self.observers.append(observer)
queue_timeout = 3
# start_time = time.clock()
try:
while True:
if ltk.check_connection.check_for_connection():
self.poll_remote()
current_timeout = self.timeout
while len(self.watch_queue) and current_timeout > 0:
self.process_queue()
time.sleep(queue_timeout)
current_timeout -= queue_timeout
time.sleep(
current_timeout) # default 60 sec total, 3 already taken
except KeyboardInterrupt:
for observer in self.observers:
observer.stop()
# except Exception as err:
# restart("Error: "+str(err)+"\nRestarting watch.")
for observer in self.observers:
observer.join()
def _on_created(self, event):
# get path
# add action
try:
db_entries = self.doc_manager.get_all_entries()
in_db = False
fn = ''
for entry in db_entries:
if event.src_path.endswith(entry['file_name']):
fn = entry['file_name']
in_db = True
if not event.is_directory and in_db:
self._on_modified(event)
else:
file_path = event.src_path
# if created file was a downloaded translation, don't add to poll. Prevents recursion from downloaded translation files when ltk watch is running
if file_path in self.download_file_paths:
self.download_file_paths.remove(file_path)
return
# if it's a hidden document, don't do anything
if not self.is_hidden_file(
file_path) and not self.is_translation(file_path):
relative_path = file_path.replace(self.path, '')
title = os.path.basename(os.path.normpath(file_path))
curr_ext = os.path.splitext(file_path)[1]
# return if the extension should be ignored or if the path is not a file
if curr_ext in self.ignore_ext or not os.path.isfile(
file_path):
# logger.info("Detected a file with an extension in the ignore list, ignoring..")
return
# only add or update the document if it's not a hidden document and it's a new file
try:
if self.doc_manager.is_doc_new(
relative_path,
self.root_path) and self.watch_folder:
#testing
#self.polled_list.add(relative_path) #test that this doesn't break other areas of watch
#end testing
self.add_document(file_path,
title,
locale=self.locale)
elif self.doc_manager.is_doc_modified(
relative_path, self.path):
self.update_content(relative_path)
else:
return
except KeyboardInterrupt:
for observer in self.observers:
observer.stop()
except ConnectionError:
print("Could not connect to remote server.")
restart()
except ValueError:
print(sys.exc_info()[1])
restart()
doc = self.doc_manager.get_doc_by_prop(
'file_name', relative_path)
if doc:
document_id = doc['id']
else:
return
if self.locale_delimiter:
try:
# curr_locale = title.split(self.locale_delimiter)[1]
# todo locale detection needs to be more robust
curr_locale = title.split(
self.locale_delimiter)[-2]
fixed_locale = map_locale(curr_locale)
if fixed_locale:
print("fixed locale: ", fixed_locale)
# self.watch_locales.add(fixed_locale)
self.detected_locales[
document_id] = fixed_locale
else:
logger.warning(
'This document\'s detected locale: {0} is not supported.'
.format(curr_locale))
except IndexError:
logger.warning(
'Cannot detect locales from file: {0}, not adding any locales'
.format(title))
self.watch_add_target(relative_path, document_id)
# logger.info('Added new document {0}'.format(title
# else:
# print("Skipping hidden file "+file_path)
except KeyboardInterrupt:
for observer in self.observers:
observer.stop()
def handleError(self):
if self.watch:
logger.warning("Could not connect to Lingotek")
restart("Restarting watch", self.timeout)
else:
raise ConnectionFailed("Could not connect to Lingotek")
def handleError(self):
if self.watch:
logger.warning("Could not connect to Lingotek")
restart("Restarting watch", self.timeout)
else:
raise ConnectionFailed("Could not connect to Lingotek")