class CSVConverter:
def __init__(self):
self.log = Logger()
def read_data(self, csv_file):
data = []
try:
reader = csv.reader(open(csv_file, 'rb'))
header = reader.next()
size = len(header)
for line in reader:
items = {}
entry_size = len(line)
for x in range(size):
if x < entry_size:
item = {header[x]: line[x]}
else:
item = {header[x]: ""}
items.update(item)
data.append(items)
except IOError:
self.log.error("Can not find csv file: %s"%csv_file)
return data
def convert_to_json(self, filepath, locale_map, comments_header):
data = self.read_data(filepath)
srclocales = []
srclocales.append('en-US')
entries = []
targetlocales = []
for item in data:
comments = []
terms = []
for header in comments_header:
if item.has_key(header):
comments.append(item.pop(header))
for key in item.keys():
if key == 'en':
term = {'locale':'en-US', 'content':item[key], 'comments':comments}
else:
if key in locale_map:
locale = locale_map[key]
else:
locale = key
term = {'locale':locale, 'content':item[key], 'comments':[]}
if key not in targetlocales:
targetlocales.append(key)
terms.append(term)
entry = {'srcLang': 'en-US', 'glossaryTerms': terms, 'sourcereference': ''}
entries.append(entry)
glossary = {'sourceLocales':srclocales, 'glossaryEntries':entries, 'targetLocales':targetlocales}
return json.dumps(glossary)
def read_project_config(self, filename):
log = Logger()
if os.path.getsize(filename) == 0:
log.info('The project config file is empty, need command line options')
return project_config
xmldoc = minidom.parse(filename)
# Read the project url
if xmldoc.getElementsByTagName("url"):
node = xmldoc.getElementsByTagName("url")[0]
project_config['url'] = getCombinedTextChildren(node)
# Read the project id
if xmldoc.getElementsByTagName("project"):
node = xmldoc.getElementsByTagName("project")[0]
project_config['project_id'] = getCombinedTextChildren(node)
# Read the project-version
if xmldoc.getElementsByTagName("project-version"):
node = xmldoc.getElementsByTagName("project-version")[0]
project_config['project_version'] = getCombinedTextChildren(node)
# Read the project-type
if xmldoc.getElementsByTagName("project-type"):
node = xmldoc.getElementsByTagName("project-type")[0]
project_config['project_type'] = getCombinedTextChildren(node)
# Read the locale map
if xmldoc.getElementsByTagName("locales"):
locales = xmldoc.getElementsByTagName("locales")[0]
localelist = locales.getElementsByTagName("locale")
project_config['locale_map'] = {}
for locale in localelist:
for node in locale.childNodes:
if node.nodeType == node.TEXT_NODE:
if locale.getAttribute("map-from"):
locale_map = {str(locale.getAttribute("map-from")): str(node.data)}
else:
locale_map = {str(node.data): str(node.data)}
project_config['locale_map'].update(locale_map)
# Read <src-dir> and <trans-dir>
if xmldoc.getElementsByTagName("src-dir"):
node = xmldoc.getElementsByTagName("src-dir")[0]
project_config['srcdir'] = getCombinedTextChildren(node)
if xmldoc.getElementsByTagName("trans-dir"):
node = xmldoc.getElementsByTagName("trans-dir")[0]
project_config['transdir'] = getCombinedTextChildren(node)
return dict((node, value.strip() if isinstance(value, str) else value)
for node, value in project_config.items() if value)
def read_project_config(self, filename):
log = Logger()
if os.path.getsize(filename) == 0:
log.info(
'The project config file is empty, need command line options')
return project_config
xmldoc = minidom.parse(filename)
# Read the project url
if xmldoc.getElementsByTagName("url"):
node = xmldoc.getElementsByTagName("url")[0]
project_config['project_url'] = getCombinedTextChildren(node)
# Read the project id
if xmldoc.getElementsByTagName("project"):
node = xmldoc.getElementsByTagName("project")[0]
project_config['project_id'] = getCombinedTextChildren(node)
# Read the project-version
if xmldoc.getElementsByTagName("project-version"):
node = xmldoc.getElementsByTagName("project-version")[0]
project_config['project_version'] = getCombinedTextChildren(node)
# Read the project-type
if xmldoc.getElementsByTagName("project-type"):
node = xmldoc.getElementsByTagName("project-type")[0]
project_config['project_type'] = getCombinedTextChildren(node)
# Read the locale map
if xmldoc.getElementsByTagName("locales"):
locales = xmldoc.getElementsByTagName("locales")[0]
localelist = locales.getElementsByTagName("locale")
for locale in localelist:
for node in locale.childNodes:
if node.nodeType == node.TEXT_NODE:
if locale.getAttribute("map-from"):
locale_map = {
str(locale.getAttribute("map-from")):
str(node.data)
}
else:
locale_map = {str(node.data): str(node.data)}
project_config['locale_map'].update(locale_map)
return project_config
class ZanataCommand:
def __init__(self):
self.log = Logger()
##############################################
##
## Commands for interaction with zanata server
##
##############################################
def check_project(self, zanataclient, command_options, project_config):
project_id = ''
iteration_id = ''
if command_options.has_key('project_id'):
project_id = command_options['project_id'][0]['value']
else:
if project_config.has_key('project_id'):
project_id = project_config['project_id']
if command_options.has_key('project_version'):
iteration_id = command_options['project_version'][0]['value']
else:
if project_config.has_key('project_version'):
iteration_id = project_config['project_version']
if not project_id:
self.log.error("Please specify a valid project id in zanata.xml or with '--project-id' option")
sys.exit(1)
if not iteration_id:
self.log.error("Please specify a valid version id in zanata.xml or with '--project-version' option")
sys.exit(1)
self.log.info("Project: %s"%project_id)
self.log.info("Version: %s"%iteration_id)
try:
zanataclient.projects.get(project_id)
except NoSuchProjectException, e:
self.log.error(e.msg)
sys.exit(1)
try:
zanataclient.projects.iterations.get(project_id, iteration_id)
return project_id, iteration_id
except NoSuchProjectException, e:
self.log.error(e.msg)
sys.exit(1)
def read_project_config(self, filename):
log = Logger()
if os.path.getsize(filename) == 0:
log.info('The project config file is empty, need command line options')
return project_config
xmldoc = minidom.parse(filename)
# Read the project url
if xmldoc.getElementsByTagName("url"):
node = xmldoc.getElementsByTagName("url")[0]
project_config['project_url'] = getCombinedTextChildren(node)
# Read the project id
if xmldoc.getElementsByTagName("project"):
node = xmldoc.getElementsByTagName("project")[0]
project_config['project_id'] = getCombinedTextChildren(node)
# Read the project-version
if xmldoc.getElementsByTagName("project-version"):
node = xmldoc.getElementsByTagName("project-version")[0]
project_config['project_version'] = getCombinedTextChildren(node)
# Read the project-type
if xmldoc.getElementsByTagName("project-type"):
node = xmldoc.getElementsByTagName("project-type")[0]
project_config['project_type'] = getCombinedTextChildren(node)
# Read the locale map
if xmldoc.getElementsByTagName("locales"):
locales = xmldoc.getElementsByTagName("locales")[0]
localelist = locales.getElementsByTagName("locale")
for locale in localelist:
for node in locale.childNodes:
if node.nodeType == node.TEXT_NODE:
if locale.getAttribute("map-from"):
locale_map = {str(locale.getAttribute("map-from")): str(node.data)}
else:
locale_map = {str(node.data): str(node.data)}
project_config['locale_map'].update(locale_map)
return project_config
class CSVConverter:
def __init__(self):
self.log = Logger()
def read_data(self, csv_file):
data = []
try:
reader = csv.reader(open(csv_file, 'rb'))
header = reader.next()
size = len(header)
for line in reader:
items = {}
entry_size = len(line)
for x in range(size):
if x < entry_size:
item = {header[x]: line[x]}
else:
item = {header[x]: ""}
items.update(item)
data.append(items)
except IOError:
self.log.error("Can not find csv file: %s" % csv_file)
return data
def read_csv_file(self, csv_file):
data = []
try:
reader = csv.reader(open(csv_file, 'rb'))
data = [line for line in reader]
except IOError:
self.log.error("Can not find csv file: %s" % csv_file)
return data
def convert_to_json(self, filepath, locale_map, comments_header):
data = self.read_csv_file(expanduser(filepath))
srclocales = []
# srclocales.append('en-US')
entries = []
targetlocales = []
csv_locales = []
comments = []
for index, item in enumerate(data):
terms = []
if index == 0:
# Assuming last two names refers to column names,for example consider following csv file
# en-US,es,ko,ru,pos,description
# Hello,Hola,test,111,noun,Greeting
# first line always contains locales and last two specifies column names
comments = [comm for comm in item[-2:]]
csv_locales = [lc for lc in item[:-2]]
continue
else:
glossary_len = len(item)
csv_locales_len = len(csv_locales)
comments_len = len(comments)
if glossary_len != csv_locales_len + comments_len:
print "Wrong entries in csv file, please check your csv file"
print "Entry in csv file", item
sys.exit(1)
glossary_comments = item[-2:]
for j in range(csv_locales_len):
if j == 0:
term = {'locale': csv_locales[j], 'content': item[j], 'comments': glossary_comments}
else:
term = {'locale': csv_locales[j], 'content': item[j], 'comments': []}
terms.append(term)
entry = {'srcLang': 'en-US', 'glossaryTerms': terms}
entries.append(entry)
glossary = {'sourceLocales': srclocales, 'glossaryEntries': entries, 'targetLocales': targetlocales}
# glossary = {'source-locales':srclocales, 'glossary-entries':entries, 'target-locales':targetlocales}
return json.dumps(glossary)
def __init__(self, url, http_headers):
self.log = Logger()
self.zanata_resource = ZanataResource(url, http_headers)
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the
# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301, USA.
import os
import sys
import string
from zanatalib.error import NoSuchFileException
from zanatacmd import ZanataCommand
from publicanutil import PublicanUtility
from zanatalib.logger import Logger
log = Logger()
class CommandsBase(object):
"""
This is base class for all zpc commands
"""
_fields = ["args", "context_data"]
def __init__(self, *args, **kargs):
for name, val in zip(self._fields, args):
setattr(self, name, val)
for key, value in kargs.iteritems():
setattr(self, key, value)
self.zanatacmd = self.create_zanatacmd()
class ZanataCommand:
def __init__(self, url, http_headers):
self.log = Logger()
self.zanata_resource = ZanataResource(url, http_headers)
def disable_ssl_cert_validation(self):
self.zanata_resource.disable_ssl_cert_validation()
##############################################
#
# Commands for interaction with zanata server
#
##############################################
def get_file_list(self, projectid, iterationid):
return self.zanata_resource.documents.get_file_list(projectid, iterationid)
def get_server_version(self, url):
try:
content = self.zanata_resource.version.get_server_version()
if content:
server_version = content['versionNo']
return server_version
except UnAvaliableResourceException:
self.log.error("Can not retrieve the server version, server may not support the version service")
except UnavailableServiceError:
self.log.error("Service Temporarily Unavailable, stop processing!")
sys.exit(1)
"""
def check_project(self, command_options, project_config):
project_id = ''
iteration_id = ''
if command_options.has_key('project_id'):
project_id = command_options['project_id'][0]['value']
else:
if project_config.has_key('project_id'):
project_id = project_config['project_id']
if command_options.has_key('project_version'):
iteration_id = command_options['project_version'][0]['value']
else:
if project_config.has_key('project_version'):
iteration_id = project_config['project_version']
if not project_id:
self.log.error("Please specify a valid project id in zanata.xml or with '--project-id' option")
sys.exit(1)
if not iteration_id:
self.log.error("Please specify a valid version id in zanata.xml or with '--project-version' option")
sys.exit(1)
self.log.info("Project: %s"%project_id)
self.log.info("Version: %s"%iteration_id)
self.verify_project(project_id, iteration_id)
"""
def verify_project(self, project_id, version_id):
try:
self.zanata_resource.projects.get(project_id)
except NoSuchProjectException, e:
self.log.error(str(e))
sys.exit(1)
try:
self.zanata_resource.projects.iterations.get(project_id, version_id)
except NoSuchProjectException, e:
self.log.error(str(e))
sys.exit(1)
class PublicanUtility:
def __init__(self):
self.log = Logger()
def create_txtflow(self, pofile):
"""
Convert the content of the pot file to a list of text flow.
@return: the dictionary object of textflow
"""
textflows = []
for entry in pofile:
reflist = []
if entry.msgctxt:
self.log.warn("encountered msgctxt; not currently supported")
m = hashlib.md5()
m.update(entry.msgid.encode('utf-8'))
textflowId = m.hexdigest()
"""
"extensions":[{"object-type":"pot-entry-header","context":"context",
"references":["fff"],"extractedComment":"extractedComment",
"flags":["java-format"]}]
"""
extracted_comment = entry.comment
references = entry.occurrences
for ref in references:
node = ref[0]+":"+ref[1]
reflist.append(node)
flags = entry.flags
#extensions_single_comment = [{'object-type':'comment','value':'test','space':'preserve'}]
#extensions_pot_entry_header = [{"object-type":"pot-entry-header","context":"context","references":["fff"],"extractedComment":"extractedComment","flags":["java-format"]}]
extensions=[{'object-type':'comment','value':extracted_comment,'space':'preserve'}, {"object-type":"pot-entry-header","context":"","references":reflist,"extractedComment":'',"flags":flags}]
textflow = {'id': textflowId, 'lang':'en-US', 'content':entry.msgid, 'extensions':extensions, 'revision':1}
textflows.append(textflow)
return textflows
def create_txtflowtarget(self, pofile):
"""
Convert the content of the po file to a list of textflowtarget.
@return: the dictionary object of textflow
"""
obs_list=pofile.obsolete_entries()
textflowtargets = []
for entry in pofile:
if entry in obs_list:
continue
m = hashlib.md5()
m.update(entry.msgid.encode('utf-8'))
textflowId = m.hexdigest()
comment = entry.comment
if entry.msgstr:
state = "Approved"
else:
state = "New"
#need judge for fuzzy state
#create extensions
extensions = [{"object-type":"comment","value":comment,"space":"preserve"}]
# {"resId":"782f49c4e93c32403ba0b51821b38b90","state":"Approved","translator":{"email":"id","name":"name"},"content":"title:
# ttff","extensions":[{"object-type":"comment","value":"testcomment","space":"preserve"}]}
# Diable the translator to avoid issues on server side
textflowtarget = {'resId': textflowId, 'state': state, 'content':entry.msgstr,'extensions':extensions}
#Temporary fill in the admin info for translator to pass the validation, waiting for server side change
textflowtargets.append(textflowtarget)
return textflowtargets
def create_extensions(self, pofile, object_type):
"""
"extensions":[{"object-type":"po-header","comment":"comment_value", "entries":[{"key":"h1","value":"v1"}]}]
"extensions":[{"object-type":"po-target-header", "comment":"comment_value", "entries":[{"key":"ht","value":"vt1"}]}]
"""
entries = []
metadatas = pofile.ordered_metadata()
for item in metadatas:
entry = {"key":item[0], "value":item[1]}
entries.append(entry)
extensions = [{"object-type":object_type,"comment":pofile.header, "entries":entries}]
return extensions
def create_pofile(self, path):
"""
Convert the po file to a pofile object in polib.
@return: pofile object
"""
try:
po = polib.pofile(path)
except Exception:
self.log.error("Can not processing the po file")
sys.exit()
return po
def get_file_list(self, path, file_type):
final_file_list = []
root_list = os.listdir(path)
for item in root_list:
if item == '.svn':
continue
full_path = os.path.join(path, item)
if full_path.endswith(file_type):
final_file_list.append(full_path)
if os.path.isdir(full_path):
final_file_list+=self.get_file_list(full_path, file_type)
return final_file_list
def hash_match(self, message, msgid):
m = hashlib.md5()
m.update(message.msgid.encode('utf-8'))
if m.hexdigest() == msgid:
return True
else:
return False
def strip_path(self, full_path, root_path):
if root_path[-1] != "/":
root_path = root_path+'/'
filename = full_path.split(root_path)[1]
if '.' in filename:
# Strip the file name
filename = filename.split('.')[0]
return filename
def potfile_to_json(self, filepath, root_path):
"""
Parse the pot file, create the request body
@param filepath: the path of the pot file
"""
filename = self.strip_path(filepath, root_path)
pofile = self.create_pofile(filepath)
textflows = self.create_txtflow(pofile)
extensions = self.create_extensions(pofile, "po-header")
items = {'name':filename, 'contentType':'application/x-gettext', 'lang':'en-US', 'extensions':extensions, 'textFlows':textflows}
return json.dumps(items), filename
def pofile_to_json(self, filepath):
"""
Parse the po file, create the request body
@param filepath: the path of the po file
"""
pofile = self.create_pofile(filepath)
textflowtargets = self.create_txtflowtarget(pofile)
#the function for extensions have not implemented yet
extensions = self.create_extensions(pofile, "po-target-header")
items = {'links':[],'extensions':extensions, 'textFlowTargets':textflowtargets}
return json.dumps(items)
def save_to_pofile(self, path, translations, pot):
"""
Save PO file to path, based on json objects of pot and translations
@param translations: the json object of the content retrieved from server
@param path: the po folder for output
@param pot: the json object of the pot retrieved from server
"""
po = polib.POFile(fpath=path)
potcontent = json.loads(pot)
# pylint: disable-msg=E1103
textflows = potcontent.get('textFlows')
if potcontent.get('extensions'):
extensions = potcontent.get('extensions')[0]
po.header = extensions.get('comment')
for item in extensions.get('entries'):
po.metadata[item['key']]=item['value']
else:
raise InvalidPOTFileException("Error", "the extensions of Resource is empty")
for textflow in textflows:
if textflow.get('extensions'):
poentry = polib.POEntry(occurrences=None)
entry_list = textflow.get('extensions')
for entry in entry_list:
if entry.get('object-type') == 'pot-entry-header':
#PotEntryHeader
#Check the references is not empty
if entry.get('references')!=[u'']:
for item in entry.get('references'):
#in some cases, entry contains more than one reference
if ' ' in item:
reference = item.split(' ')
ref_list = []
for i in reference:
ref = tuple(i.split(':'))
ref_list.append(ref)
poentry.occurrences= ref_list
else:
poentry.occurrences = [tuple(item.split(':'))]
else:
poentry.occurrences = None
#print poentry.occurrences
poentry.flags = entry.get('flags')
if entry.get('object-type') == 'comment':
#SimpleComment
poentry.comment = entry.get('value')
poentry.msgid = textflow.get('content')
po.append(poentry)
else:
raise InvalidPOTFileException("Error", "the extensions of TextFlow is empty")
#If the translation is exist, read the content of the po file
if translations:
content = json.loads(translations)
"""
"extensions":[{"object-type":"po-target-header", "comment":"comment_value", "entries":[{"key":"ht","value":"vt1"}]}]
"""
if content.get('extensions'):
ext = content.get('extensions')[0]
header_comment = ext.get('comment')
if header_comment:
po.header = header_comment
for item in ext.get('entries'):
po.metadata[item['key']]=item['value']
targets = content.get('textFlowTargets')
"""
"extensions":[{"object-type":"comment","value":"testcomment","space":"preserve"}]
"""
# copy any other stuff you need to transfer
for message in po:
for translation in targets:
if translation.get('extensions'):
extensions=translation.get('extensions')[0]
#if extensions:
# ext_type = extensions.get('object-type')
# comment = extensions.get('comment')
# entries = extensions.get('value')
if self.hash_match(message, translation.get('resId')):
message.msgstr = translation.get('content')
# finally save resulting po to outpath as lang/myfile.po
po.save()
# pylint: disable-msg=E1103
self.log.info("Writing po file to %s"%(path))
# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301, USA.
import os
import sys
import string
from zanatalib.error import UnAvaliableResourceException
from zanatalib.error import NoSuchFileException
from zanatalib.error import UnavailableServiceError
from zanatacmd import ZanataCommand
from parseconfig import ZanataConfig
from publicanutil import PublicanUtility
from zanatalib.logger import Logger
log = Logger()
class Push:
def read_project_config(self, command_options):
project_config = {}
config = ZanataConfig()
#Read the project configuration file using --project-config option
config_file = [os.path.join(os.getcwd(), filename) for filename\
in ['zanata.xml', 'flies.xml']]
if command_options.has_key('project_config'):
config_file.append(command_options['project_config'][0]['value'])
for path in config_file:
if os.path.exists(path):
log.info("Loading zanata project config from: %s" % path)
# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301, USA.
import os
import sys
import string
from zanatalib.error import UnAvaliableResourceException
from zanatalib.error import NoSuchFileException
from zanatalib.error import UnavailableServiceError
from zanatacmd import ZanataCommand
from parseconfig import ZanataConfig
from publicanutil import PublicanUtility
from zanatalib.logger import Logger
log = Logger()
class Push(object):
_fields = ["command_options", "args", "project_type", "http_headers"]
def __init__(self, *args, **kargs):
for name, val in zip(self._fields, args):
setattr(self, name, val)
for key, value in kargs.iteritems():
setattr(self, key, value)
url, self.project_id, self.version_id, self.project_config = self.get_projectinfo(self.command_options)
self.zanatacmd, username, client_version, server_version = self.create_zanatacmd(
url, self.command_options, self.http_headers
)
self.plural_support = self.check_plural_support(server_version)
class OptionsUtil:
def __init__(self, options):
self.command_options = options
self.project_config = {}
self.log = Logger()
self.config = ZanataConfig()
def apply_configfiles(self):
url = self.apply_project_config()
username, apikey = self.apply_user_config(url)
# The value in commandline options will overwrite the value in user-config file
if self.command_options.has_key("user_name"):
username = self.command_options["user_name"][0]["value"]
if self.command_options.has_key("key"):
apikey = self.command_options["key"][0]["value"]
self.log.info("zanata server: %s" % url)
return url, username, apikey
def apply_project_config(self):
# Read the project configuration file using --project-config option
config_file = [os.path.join(os.getcwd(), filename) for filename in ["zanata.xml", "flies.xml"]]
if self.command_options.has_key("project_config"):
config_file.append(self.command_options["project_config"][0]["value"])
for path in config_file:
if os.path.exists(path):
self.log.info("Loading zanata project config from: %s" % path)
self.project_config = self.config.read_project_config(path)
break
if not self.project_config:
self.log.info("Can not find zanata.xml, please specify the path of zanata.xml")
# process the url of server
if self.project_config.has_key("project_url"):
url = self.project_config["project_url"]
# The value in options will override the value in project-config file
if self.command_options.has_key("url"):
self.log.info("Overriding url of server with command line option")
url = self.command_options["url"][0]["value"]
if not url or url.isspace():
self.log.error("Please specify valid server url in zanata.xml or with '--url' option")
sys.exit(1)
url = self.trim_url(url)
return url
def trim_url(self, url):
if " " in url or "\n" in url:
self.log.info("Warning, the url which contains '\\n' or whitespace is not valid, please check zanata.xml")
url = url.strip()
if url[-1] == "/":
url = url[:-1]
return url
def get_localemap(self):
if self.project_config and self.project_config.has_key("locale_map"):
locale_map = self.project_config["locale_map"]
return locale_map
def apply_user_config(self, url):
user_name = ""
apikey = ""
# Try to read user-config file
user_config = [
os.path.join(os.path.expanduser("~") + "/.config", filename) for filename in ["zanata.ini", "flies.ini"]
]
if self.command_options.has_key("user_config"):
user_config.append(self.command_options["user_config"][0]["value"])
for path in user_config:
if os.path.exists(path):
self.log.info("Loading zanata user config from: %s" % path)
# Read the user-config file
self.config.set_userconfig(path)
try:
server = self.config.get_server(url)
if server:
user_name = self.config.get_config_value("username", "servers", server)
apikey = self.config.get_config_value("key", "servers", server)
except Exception, e:
self.log.info("Processing user-config file:%s" % str(e))
break
break
if not (user_name, apikey):
self.log.info("Can not find user-config file in home folder, current path or path in 'user-config' option")
return (user_name, apikey)
class ZanataCommand:
def __init__(self, url, username=None, apikey=None, http_headers=None):
self.log = Logger()
self.zanata_resource = ZanataResource(url, username, apikey,
http_headers)
def disable_ssl_cert_validation(self):
self.zanata_resource.disable_ssl_cert_validation()
##############################################
#
# Commands for interaction with zanata server
#
##############################################
def get_file_list(self, projectid, iterationid):
return self.zanata_resource.documents.get_file_list(
projectid, iterationid)
def get_server_version(self, url):
try:
content = self.zanata_resource.version.get_server_version()
if content:
server_version = content['versionNo']
return server_version
except UnAvaliableResourceException:
self.log.error(
"Can not retrieve the server version, server may not support the version service"
)
except UnavailableServiceError:
self.log.error("Service Temporarily Unavailable, stop processing!")
sys.exit(1)
"""
def check_project(self, command_options, project_config):
project_id = ''
iteration_id = ''
if command_options.has_key('project_id'):
project_id = command_options['project_id'][0]['value']
else:
if project_config.has_key('project_id'):
project_id = project_config['project_id']
if command_options.has_key('project_version'):
iteration_id = command_options['project_version'][0]['value']
else:
if project_config.has_key('project_version'):
iteration_id = project_config['project_version']
if not project_id:
self.log.error("Please specify a valid project id in zanata.xml or with '--project-id' option")
sys.exit(1)
if not iteration_id:
self.log.error("Please specify a valid version id in zanata.xml or with '--project-version' option")
sys.exit(1)
self.log.info("Project: %s"%project_id)
self.log.info("Version: %s"%iteration_id)
self.verify_project(project_id, iteration_id)
"""
def verify_project(self, project_id, version_id):
try:
self.zanata_resource.projects.get(project_id)
except NoSuchProjectException, e:
self.log.error(str(e))
sys.exit(1)
try:
self.zanata_resource.projects.iterations.get(
project_id, version_id)
except NoSuchProjectException, e:
self.log.error(str(e))
sys.exit(1)
def __init__(self, url, username = None, apikey = None,http_headers=None):
self.log = Logger()
self.zanata_resource = ZanataResource(url, username, apikey,http_headers)
from zanatalib.versionservice import VersionService
from zanatalib.client import ZanataResource
from zanatalib.error import UnAvaliableResourceException
from zanatalib.error import NoSuchFileException
from zanatalib.error import UnavailableServiceError
from zanatalib.logger import Logger
from zanatacmd import ZanataCommand
from parseconfig import ZanataConfig
from publicanutil import PublicanUtility
from command import makeHandler
from command import strip_docstring
from command import parse_command_line
from command import handle_program
log = Logger()
option_sets = {
'url': [
dict(
type='command',
long=['--url'],
metavar='URL',
),
],
'user_name': [
dict(
type='command',
long=['--username'],
metavar='USERNAME',
),
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the
# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301, USA.
import os
import sys
import string
from zanatalib.logger import Logger
from pushcmd import Push
log = Logger()
class GenericPull(Push):
def __init__(self, *args, **kargs):
super(GenericPull, self).__init__(*args, **kargs)
def process_transdir(self, command_options, src_folder):
trans_folder = ""
if command_options.has_key('transdir'):
trans_folder = command_options['transdir'][0]['value']
elif src_folder:
trans_folder = src_folder
else:
trans_folder = os.getcwd()
def read_project_config(self, filename):
log = Logger()
if os.path.getsize(filename) == 0:
log.info('The project config file is empty, need command line options')
return project_config
xmldoc = minidom.parse(filename)
#Read the project url
if xmldoc.getElementsByTagName("url"):
node = xmldoc.getElementsByTagName("url")[0]
rc = ""
for node in node.childNodes:
if node.nodeType in ( node.TEXT_NODE, node.CDATA_SECTION_NODE):
rc = rc + node.data
project_config['project_url'] = rc
#Read the project id
if xmldoc.getElementsByTagName("project"):
node = xmldoc.getElementsByTagName("project")[0]
rc = ""
for node in node.childNodes:
if node.nodeType in ( node.TEXT_NODE, node.CDATA_SECTION_NODE):
rc = rc + node.data
project_config['project_id'] = rc
#Read the project-version
if xmldoc.getElementsByTagName("project-version"):
node = xmldoc.getElementsByTagName("project-version")[0]
rc = ""
for node in node.childNodes:
if node.nodeType in ( node.TEXT_NODE, node.CDATA_SECTION_NODE):
rc = rc + node.data
project_config['project_version'] = rc
if xmldoc.getElementsByTagName("python-client"):
python_config = xmldoc.getElementsByTagName("python-client")[0]
srcdir = python_config.getElementsByTagName("dir")
rc = ""
for folder in srcdir:
for node in folder.childNodes:
if node.nodeType in ( node.TEXT_NODE, node.CDATA_SECTION_NODE):
rc = rc + node.data
project_config['project_srcdir'] = rc
#Read the locale map
if xmldoc.getElementsByTagName("locales"):
locales = xmldoc.getElementsByTagName("locales")[0]
localelist = locales.getElementsByTagName("locale")
for locale in localelist:
for node in locale.childNodes:
if node.nodeType == node.TEXT_NODE:
if locale.getAttribute("map-from"):
locale_map = {locale.getAttribute("map-from"):node.data}
project_config['locale_map'].update(locale_map)
else:
locale_map = {node.data:node.data}
project_config['locale_map'].update(locale_map)
return project_config
def __init__(self):
self.log = Logger()
class PublicanUtility:
def __init__(self):
self.log = Logger()
def create_txtflow(self, pofile):
"""
Convert the content of the pot file to a list of text flow.
@return: the dictionary object of textflow
"""
textflows = []
for entry in pofile:
context = None
reflist = []
content = ""
if entry.msgctxt is not None:
hashbase = entry.msgctxt + u"\u0000" + entry.msgid
context = entry.msgctxt
else:
hashbase = entry.msgid
# pylint: disable=E1101
m = hashlib.md5()
m.update(hashbase.encode('utf-8'))
textflowId = m.hexdigest()
"""
"extensions":[{"object-type":"pot-entry-header","context":"context",
"references":["fff"],"extractedComment":"extractedComment",
"flags":["java-format"]}]
"""
extracted_comment = entry.comment
references = entry.occurrences
for ref in references:
node = ref[0] + ":" + ref[1]
reflist.append(node)
flags = entry.flags
if entry.msgid_plural:
content = [entry.msgid, entry.msgid_plural]
else:
content = entry.msgid
if context is not None:
extensions = [{'object-type': 'comment', 'value': extracted_comment, 'space': 'preserve'},
{"object-type": "pot-entry-header", "context": context, "references": reflist,
"extractedComment": '', "flags": flags}]
else:
extensions = [{'object-type': 'comment', 'value': extracted_comment, 'space': 'preserve'},
{"object-type": "pot-entry-header", "references": reflist, "extractedComment": '',
"flags": flags}]
if entry.msgid_plural:
textflow = {'id': textflowId, 'lang': 'en-US', 'contents': content, 'plural': 'true', 'extensions': extensions}
else:
textflow = {'id': textflowId, 'lang': 'en-US', 'content': content, 'plural': 'false', 'extensions': extensions}
textflows.append(textflow)
return textflows
def check_empty(self, contents):
for string in contents:
if string != u'':
return False
return True
def check_nonempty(self, contents):
for string in contents:
if string == u'':
return False
return True
def get_contentstate(self, entry):
"""
Determine the ContentState for a PO entry, based on contents and fuzzy flag
@return: the state
"""
fuzzy = False
contents = []
if "fuzzy" in entry.flags:
fuzzy = True
if entry.msgid_plural:
keys = entry.msgstr_plural.keys()
keys.sort()
for key in keys:
contents.append(entry.msgstr_plural[key])
else:
contents.append(entry.msgstr)
if self.check_empty(contents):
fuzzy = False
if fuzzy:
return "NeedReview"
if self.check_nonempty(contents):
return "Approved"
else:
return "New"
def create_txtflowtarget(self, pofile):
"""
Convert the content of the po file to a list of textflowtarget.
@return: the dictionary object of textflow
"""
obs_list = pofile.obsolete_entries()
textflowtargets = []
content = ""
for entry in pofile:
if entry in obs_list:
continue
if entry.msgctxt is not None:
hashbase = entry.msgctxt + u"\u0000" + entry.msgid
else:
hashbase = entry.msgid
# pylint: disable=E1101
m = hashlib.md5()
m.update(hashbase.encode('utf-8'))
textflowId = m.hexdigest()
translator_comment = entry.tcomment
state = self.get_contentstate(entry)
# create extensions
extensions = [{"object-type": "comment", "value": translator_comment, "space": "preserve"}]
if entry.msgid_plural:
content = []
keys = entry.msgstr_plural.keys()
keys.sort()
for key in keys:
content.append(entry.msgstr_plural[key])
textflowtarget = {'resId': textflowId, 'state': state, 'contents': content, 'extensions': extensions}
else:
content = entry.msgstr
textflowtarget = {'resId': textflowId, 'state': state, 'content': content, 'extensions': extensions}
textflowtargets.append(textflowtarget)
return textflowtargets
def validate_content_type(self, content_type, object_type):
PATTERN = r'.+? charset=([\w_\-:\.]+)'
rxt = re.compile(unicode(PATTERN))
match = rxt.search(content_type)
if match:
enc = match.group(1).strip()
if enc not in ["UTF-8", "utf-8", "utf8", "ascii", "UTF8", "ASCII"]:
if enc == 'CHARSET':
if object_type == 'po-target-header':
self.log.error("Invalid encoding CHARSET; please correct the Content-Type charset (UTF-8 recommended)")
sys.exit(1)
else:
self.log.error("Unsupported encoding; please change the Content-Type charset (UTF-8 recommended)")
sys.exit(1)
def create_extensions(self, pofile, object_type):
"""
"extensions":[{"object-type":"po-header","comment":"comment_value", "entries":[{"key":"h1","value":"v1"}]}]
"extensions":[{"object-type":"po-target-header", "comment":"comment_value", "entries":[{"key":"ht","value":"vt1"}]}]
"""
entries = []
metadatas = pofile.ordered_metadata()
for item in metadatas:
entry = {"key": item[0], "value": item[1]}
entries.append(entry)
if pofile.metadata.has_key('Content-Type'):
self.validate_content_type(pofile.metadata['Content-Type'], object_type)
extensions = [{"object-type": object_type, "comment": pofile.header, "entries": entries}]
return extensions
def create_pofile(self, path):
"""
Convert the po file to a pofile object in polib.
@return: pofile object
"""
try:
po = polib.pofile(path)
except Exception, e:
self.log.error(str(e))
sys.exit(1)
return po
from zanatalib.logger import Logger
from zanatacmd import ZanataCommand
from parseconfig import ZanataConfig
from publicanutil import PublicanUtility
from optionsutil import OptionsUtil
from command import makeHandler
from command import strip_docstring
from command import parse_command_line
from command import handle_program
from pushcmd import PoPush
from pushcmd import PublicanPush
from pushcmd import GenericPush
from pullcmd import GenericPull
log = Logger()
option_sets = {
'url': [
dict(
type='command',
long=['--url'],
metavar='URL',
),
],
'user_name': [
dict(
type='command',
long=['--username'],
metavar='USERNAME',
),
def __init__(self, url, username=None, apikey=None, http_headers=None):
self.log = Logger()
self.zanata_resource = ZanataResource(url, username, apikey,
http_headers)
from zanatalib.logger import Logger
from zanatacmd import ZanataCommand
from parseconfig import ZanataConfig
from publicanutil import PublicanUtility
from optionsutil import OptionsUtil
from command import makeHandler
from command import strip_docstring
from command import parse_command_line
from command import handle_program
from pushcmd import PoPush
from pushcmd import PublicanPush
from pushcmd import GenericPush
from pullcmd import GenericPull
log = Logger()
option_sets = {
'url': [
dict(
type='command',
long=['--url'],
metavar='URL',
),
],
'user_name': [
dict(
type='command',
long=['--username'],
metavar='USERNAME',
),
class PublicanUtility:
def __init__(self):
self.log = Logger()
def create_txtflow(self, pofile):
"""
Convert the content of the pot file to a list of text flow.
@return: the dictionary object of textflow
"""
textflows = []
for entry in pofile:
context = None
reflist = []
content = ""
if entry.msgctxt is not None:
hashbase = entry.msgctxt + u"\u0000" + entry.msgid
context = entry.msgctxt
else:
hashbase = entry.msgid
# pylint: disable=E1101
m = hashlib.md5()
m.update(hashbase.encode('utf-8'))
textflowId = m.hexdigest()
"""
"extensions":[{"object-type":"pot-entry-header","context":"context",
"references":["fff"],"extractedComment":"extractedComment",
"flags":["java-format"]}]
"""
extracted_comment = entry.comment
references = entry.occurrences
for ref in references:
node = ref[0] + ":" + ref[1]
reflist.append(node)
flags = entry.flags
if entry.msgid_plural:
content = [entry.msgid, entry.msgid_plural]
else:
content = entry.msgid
if context is not None:
extensions = [{
'object-type': 'comment',
'value': extracted_comment,
'space': 'preserve'
}, {
"object-type": "pot-entry-header",
"context": context,
"references": reflist,
"extractedComment": '',
"flags": flags
}]
else:
extensions = [{
'object-type': 'comment',
'value': extracted_comment,
'space': 'preserve'
}, {
"object-type": "pot-entry-header",
"references": reflist,
"extractedComment": '',
"flags": flags
}]
if entry.msgid_plural:
textflow = {
'id': textflowId,
'lang': 'en-US',
'contents': content,
'plural': 'true',
'extensions': extensions
}
else:
textflow = {
'id': textflowId,
'lang': 'en-US',
'content': content,
'plural': 'false',
'extensions': extensions
}
textflows.append(textflow)
return textflows
def check_empty(self, contents):
for string in contents:
if string != u'':
return False
return True
def check_nonempty(self, contents):
for string in contents:
if string == u'':
return False
return True
def get_contentstate(self, entry):
"""
Determine the ContentState for a PO entry, based on contents and fuzzy flag
@return: the state
"""
fuzzy = False
contents = []
if "fuzzy" in entry.flags:
fuzzy = True
if entry.msgid_plural:
keys = entry.msgstr_plural.keys()
keys.sort()
for key in keys:
contents.append(entry.msgstr_plural[key])
else:
contents.append(entry.msgstr)
if self.check_empty(contents):
fuzzy = False
if fuzzy:
return "NeedReview"
if self.check_nonempty(contents):
return "Approved"
else:
return "New"
def create_txtflowtarget(self, pofile):
"""
Convert the content of the po file to a list of textflowtarget.
@return: the dictionary object of textflow
"""
obs_list = pofile.obsolete_entries()
textflowtargets = []
content = ""
for entry in pofile:
if entry in obs_list:
continue
if entry.msgctxt is not None:
hashbase = entry.msgctxt + u"\u0000" + entry.msgid
else:
hashbase = entry.msgid
# pylint: disable=E1101
m = hashlib.md5()
m.update(hashbase.encode('utf-8'))
textflowId = m.hexdigest()
translator_comment = entry.tcomment
state = self.get_contentstate(entry)
#create extensions
extensions = [{
"object-type": "comment",
"value": translator_comment,
"space": "preserve"
}]
if entry.msgid_plural:
content = []
keys = entry.msgstr_plural.keys()
keys.sort()
for key in keys:
content.append(entry.msgstr_plural[key])
textflowtarget = {
'resId': textflowId,
'state': state,
'contents': content,
'extensions': extensions
}
else:
content = entry.msgstr
textflowtarget = {
'resId': textflowId,
'state': state,
'content': content,
'extensions': extensions
}
textflowtargets.append(textflowtarget)
return textflowtargets
def validate_content_type(self, content_type, object_type):
PATTERN = r'.+? charset=([\w_\-:\.]+)'
rxt = re.compile(unicode(PATTERN))
match = rxt.search(content_type)
if match:
enc = match.group(1).strip()
if enc not in ["UTF-8", "utf-8", "utf8", "ascii", "UTF8", "ASCII"]:
if enc == 'CHARSET':
if object_type == 'po-target-header':
self.log.error(
"Invalid encoding CHARSET; please correct the Content-Type charset (UTF-8 recommended)"
)
sys.exit(1)
else:
self.log.error(
"Unsupported encoding; please change the Content-Type charset (UTF-8 recommended)"
)
sys.exit(1)
def create_extensions(self, pofile, object_type):
"""
"extensions":[{"object-type":"po-header","comment":"comment_value", "entries":[{"key":"h1","value":"v1"}]}]
"extensions":[{"object-type":"po-target-header", "comment":"comment_value", "entries":[{"key":"ht","value":"vt1"}]}]
"""
entries = []
metadatas = pofile.ordered_metadata()
for item in metadatas:
entry = {"key": item[0], "value": item[1]}
entries.append(entry)
if pofile.metadata.has_key('Content-Type'):
self.validate_content_type(pofile.metadata['Content-Type'],
object_type)
extensions = [{
"object-type": object_type,
"comment": pofile.header,
"entries": entries
}]
return extensions
def create_pofile(self, path):
"""
Convert the po file to a pofile object in polib.
@return: pofile object
"""
try:
po = polib.pofile(path)
except Exception, e:
self.log.error(str(e))
sys.exit(1)
return po
def __init__(self, options):
self.command_options = options
self.project_config = {}
self.log = Logger()
self.config = ZanataConfig()
class OptionsUtil:
def __init__(self, options):
self.command_options = options
self.project_config = {}
self.log = Logger()
self.config = ZanataConfig()
def apply_configfiles(self):
url = self.apply_project_config()
username, apikey = self.apply_user_config(url)
# The value in commandline options will overwrite the value in user-config file
if self.command_options.has_key('user_name'):
username = self.command_options['user_name'][0]['value']
if self.command_options.has_key('key'):
apikey = self.command_options['key'][0]['value']
self.log.info("zanata server: %s" % url)
return url, username, apikey
def apply_project_config(self):
url = ""
# Read the project configuration file using --project-config option
config_file = [os.path.join(os.getcwd(), filename) for filename
in ['zanata.xml', 'flies.xml']]
if self.command_options.has_key('project_config'):
config_file.append(self.command_options['project_config'][0]['value'])
for path in config_file:
if os.path.exists(path):
self.log.info("Loading zanata project config from: %s" % path)
self.project_config = self.config.read_project_config(path)
break
if not self.project_config:
self.log.info("Can not find zanata.xml, please specify the path of zanata.xml")
# process the url of server
if self.project_config.has_key('project_url'):
url = self.project_config['project_url']
# The value in options will override the value in project-config file
if self.command_options.has_key('url'):
self.log.info("Overriding url of server with command line option")
url = self.command_options['url'][0]['value']
if not url or url.isspace():
self.log.error("Please specify valid server url in zanata.xml or with '--url' option")
sys.exit(1)
url = self.trim_url(url)
return url
def trim_url(self, url):
if ' ' in url or '\n' in url:
self.log.info("Warning, the url which contains '\\n' or whitespace is not valid, please check zanata.xml")
url = url.strip()
if url[-1] == "/":
url = url[:-1]
return url
def get_localemap(self):
if self.project_config and self.project_config.has_key('locale_map'):
locale_map = self.project_config['locale_map']
return locale_map
def apply_user_config(self, url):
user_name = ""
apikey = ""
# Try to read user-config file
user_config = [os.path.join(os.path.expanduser("~") + '/.config', filename) for filename in ['zanata.ini', 'flies.ini']]
if self.command_options.has_key('user_config'):
user_config.append(self.command_options['user_config'][0]['value'])
for path in user_config:
if os.path.exists(path):
self.log.info("Loading zanata user config from: %s" % path)
# Read the user-config file
self.config.set_userconfig(path)
try:
server = self.config.get_server(url)
if server:
user_name = self.config.get_config_value("username", "servers", server)
apikey = self.config.get_config_value("key", "servers", server)
except Exception, e:
self.log.info("Processing user-config file:%s" % str(e))
break
break
if not (user_name, apikey):
self.log.info("Can not find user-config file in home folder, current path or path in 'user-config' option")
return (user_name, apikey)
def __init__(self):
self.log = Logger()
def __init__(self, options):
self.command_options = options
self.project_config = {}
self.log = Logger()
self.config = ZanataConfig()
from zanatalib.client import ZanataResource
from zanatalib.error import UnAvaliableResourceException
from zanatalib.error import NoSuchFileException
from zanatalib.error import UnavailableServiceError
from zanatalib.logger import Logger
from zanatacmd import ZanataCommand
from parseconfig import ZanataConfig
from publicanutil import PublicanUtility
from optionsutil import OptionsUtil
from command import makeHandler
from command import strip_docstring
from command import parse_command_line
from command import handle_program
log = Logger()
option_sets = {
'url': [
dict(
type='command',
long=['--url'],
metavar='URL',
),
],
'user_name': [
dict(
type='command',
long=['--username'],
metavar='USERNAME',
),
# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301, USA.
import os
import sys
import string
from zanatalib.error import UnAvaliableResourceException
from zanatalib.error import NoSuchFileException
from zanatalib.error import UnavailableServiceError
from zanatacmd import ZanataCommand
from parseconfig import ZanataConfig
from publicanutil import PublicanUtility
from zanatalib.logger import Logger
log = Logger()
class Push(object):
_fields = ['command_options', 'args', 'project_type', 'http_headers']
def __init__(self, *args, **kargs):
for name, val in zip(self._fields, args):
setattr(self, name, val)
for key, value in kargs.iteritems():
setattr(self, key, value)
url, self.project_id, self.version_id, self.project_config = self.get_projectinfo(
self.command_options)
self.zanatacmd, username, client_version, server_version = self.create_zanatacmd(
url, self.command_options, self.http_headers)
self.plural_support = self.check_plural_support(server_version)
