class CSVConverter:
def __init__(self):
self.log = Logger()
def read_data(self, csv_file):
data = []
try:
reader = csv.reader(open(csv_file, 'rb'))
header = reader.next()
size = len(header)
for line in reader:
items = {}
entry_size = len(line)
for x in range(size):
if x < entry_size:
item = {header[x]: line[x]}
else:
item = {header[x]: ""}
items.update(item)
data.append(items)
except IOError:
self.log.error("Can not find csv file: %s"%csv_file)
return data
def convert_to_json(self, filepath, locale_map, comments_header):
data = self.read_data(filepath)
srclocales = []
srclocales.append('en-US')
entries = []
targetlocales = []
for item in data:
comments = []
terms = []
for header in comments_header:
if item.has_key(header):
comments.append(item.pop(header))
for key in item.keys():
if key == 'en':
term = {'locale':'en-US', 'content':item[key], 'comments':comments}
else:
if key in locale_map:
locale = locale_map[key]
else:
locale = key
term = {'locale':locale, 'content':item[key], 'comments':[]}
if key not in targetlocales:
targetlocales.append(key)
terms.append(term)
entry = {'srcLang': 'en-US', 'glossaryTerms': terms, 'sourcereference': ''}
entries.append(entry)
glossary = {'sourceLocales':srclocales, 'glossaryEntries':entries, 'targetLocales':targetlocales}
return json.dumps(glossary)
class ZanataCommand:
def __init__(self):
self.log = Logger()
##############################################
##
## Commands for interaction with zanata server
##
##############################################
def check_project(self, zanataclient, command_options, project_config):
project_id = ''
iteration_id = ''
if command_options.has_key('project_id'):
project_id = command_options['project_id'][0]['value']
else:
if project_config.has_key('project_id'):
project_id = project_config['project_id']
if command_options.has_key('project_version'):
iteration_id = command_options['project_version'][0]['value']
else:
if project_config.has_key('project_version'):
iteration_id = project_config['project_version']
if not project_id:
self.log.error("Please specify a valid project id in zanata.xml or with '--project-id' option")
sys.exit(1)
if not iteration_id:
self.log.error("Please specify a valid version id in zanata.xml or with '--project-version' option")
sys.exit(1)
self.log.info("Project: %s"%project_id)
self.log.info("Version: %s"%iteration_id)
try:
zanataclient.projects.get(project_id)
except NoSuchProjectException, e:
self.log.error(e.msg)
sys.exit(1)
try:
zanataclient.projects.iterations.get(project_id, iteration_id)
return project_id, iteration_id
except NoSuchProjectException, e:
self.log.error(e.msg)
sys.exit(1)
if args:
try:
full_path = search_file(tmlfolder, args[0])
filelist.append(full_path)
except NoSuchFileException, e:
log.error(e.msg)
sys.exit(1)
else:
if not command_options.has_key('srcfile'):
#get all the pot files from the template folder
publicanutil = PublicanUtility()
filelist = publicanutil.get_file_list(tmlfolder, ".pot")
if not filelist:
log.error("The template folder is empty")
sys.exit(1)
if command_options.has_key('force'):
force = True
zanatacmd.del_server_content(zanata, tmlfolder, project_id, iteration_id, filelist, force, "gettext")
if importpo:
zanatacmd.push_command(zanata, filelist, tmlfolder, project_id, iteration_id, copytrans, plural_support, import_param)
else:
zanatacmd.push_command(zanata, filelist, tmlfolder, project_id, iteration_id, copytrans, plural_support)
def publican_pull(command_options, args):
"""
Usage: zanata publican pull [OPTIONS] {documents} {lang}
sys.exit(1)
if args:
try:
full_path = self.search_file(tmlfolder, args[0])
filelist.append(full_path)
except NoSuchFileException, e:
log.error(e.msg)
sys.exit(1)
else:
#get all the pot files from the template folder
publicanutil = PublicanUtility()
filelist = publicanutil.get_file_list(tmlfolder, ".pot")
if not filelist:
log.error("The template folder is empty")
sys.exit(1)
deletefiles = True
if command_options.has_key('force'):
force = True
if project_type == 'podir':
log.info("POT directory (originals):%s" % tmlfolder)
folder = None
elif project_type == 'gettext':
log.info("PO directory (originals):%s" % tmlfolder)
folder = tmlfolder
if pushtrans is None:
pushtrans = self.get_pushtrans(command_options)
class PublicanUtility:
def __init__(self):
self.log = Logger()
def create_txtflow(self, pofile):
"""
Convert the content of the pot file to a list of text flow.
@return: the dictionary object of textflow
"""
textflows = []
for entry in pofile:
context = None
reflist = []
content = ""
if entry.msgctxt is not None:
hashbase = entry.msgctxt + u"\u0000" + entry.msgid
context = entry.msgctxt
else:
hashbase = entry.msgid
# pylint: disable=E1101
m = hashlib.md5()
m.update(hashbase.encode('utf-8'))
textflowId = m.hexdigest()
"""
"extensions":[{"object-type":"pot-entry-header","context":"context",
"references":["fff"],"extractedComment":"extractedComment",
"flags":["java-format"]}]
"""
extracted_comment = entry.comment
references = entry.occurrences
for ref in references:
node = ref[0] + ":" + ref[1]
reflist.append(node)
flags = entry.flags
if entry.msgid_plural:
content = [entry.msgid, entry.msgid_plural]
else:
content = entry.msgid
if context is not None:
extensions = [{'object-type': 'comment', 'value': extracted_comment, 'space': 'preserve'},
{"object-type": "pot-entry-header", "context": context, "references": reflist,
"extractedComment": '', "flags": flags}]
else:
extensions = [{'object-type': 'comment', 'value': extracted_comment, 'space': 'preserve'},
{"object-type": "pot-entry-header", "references": reflist, "extractedComment": '',
"flags": flags}]
if entry.msgid_plural:
textflow = {'id': textflowId, 'lang': 'en-US', 'contents': content, 'plural': 'true', 'extensions': extensions}
else:
textflow = {'id': textflowId, 'lang': 'en-US', 'content': content, 'plural': 'false', 'extensions': extensions}
textflows.append(textflow)
return textflows
def check_empty(self, contents):
for string in contents:
if string != u'':
return False
return True
def check_nonempty(self, contents):
for string in contents:
if string == u'':
return False
return True
def get_contentstate(self, entry):
"""
Determine the ContentState for a PO entry, based on contents and fuzzy flag
@return: the state
"""
fuzzy = False
contents = []
if "fuzzy" in entry.flags:
fuzzy = True
if entry.msgid_plural:
keys = entry.msgstr_plural.keys()
keys.sort()
for key in keys:
contents.append(entry.msgstr_plural[key])
else:
contents.append(entry.msgstr)
if self.check_empty(contents):
fuzzy = False
if fuzzy:
return "NeedReview"
if self.check_nonempty(contents):
return "Approved"
else:
return "New"
def create_txtflowtarget(self, pofile):
"""
Convert the content of the po file to a list of textflowtarget.
@return: the dictionary object of textflow
"""
obs_list = pofile.obsolete_entries()
textflowtargets = []
content = ""
for entry in pofile:
if entry in obs_list:
continue
if entry.msgctxt is not None:
hashbase = entry.msgctxt + u"\u0000" + entry.msgid
else:
hashbase = entry.msgid
# pylint: disable=E1101
m = hashlib.md5()
m.update(hashbase.encode('utf-8'))
textflowId = m.hexdigest()
translator_comment = entry.tcomment
state = self.get_contentstate(entry)
# create extensions
extensions = [{"object-type": "comment", "value": translator_comment, "space": "preserve"}]
if entry.msgid_plural:
content = []
keys = entry.msgstr_plural.keys()
keys.sort()
for key in keys:
content.append(entry.msgstr_plural[key])
textflowtarget = {'resId': textflowId, 'state': state, 'contents': content, 'extensions': extensions}
else:
content = entry.msgstr
textflowtarget = {'resId': textflowId, 'state': state, 'content': content, 'extensions': extensions}
textflowtargets.append(textflowtarget)
return textflowtargets
def validate_content_type(self, content_type, object_type):
PATTERN = r'.+? charset=([\w_\-:\.]+)'
rxt = re.compile(unicode(PATTERN))
match = rxt.search(content_type)
if match:
enc = match.group(1).strip()
if enc not in ["UTF-8", "utf-8", "utf8", "ascii", "UTF8", "ASCII"]:
if enc == 'CHARSET':
if object_type == 'po-target-header':
self.log.error("Invalid encoding CHARSET; please correct the Content-Type charset (UTF-8 recommended)")
sys.exit(1)
else:
self.log.error("Unsupported encoding; please change the Content-Type charset (UTF-8 recommended)")
sys.exit(1)
def create_extensions(self, pofile, object_type):
"""
"extensions":[{"object-type":"po-header","comment":"comment_value", "entries":[{"key":"h1","value":"v1"}]}]
"extensions":[{"object-type":"po-target-header", "comment":"comment_value", "entries":[{"key":"ht","value":"vt1"}]}]
"""
entries = []
metadatas = pofile.ordered_metadata()
for item in metadatas:
entry = {"key": item[0], "value": item[1]}
entries.append(entry)
if pofile.metadata.has_key('Content-Type'):
self.validate_content_type(pofile.metadata['Content-Type'], object_type)
extensions = [{"object-type": object_type, "comment": pofile.header, "entries": entries}]
return extensions
def create_pofile(self, path):
"""
Convert the po file to a pofile object in polib.
@return: pofile object
"""
try:
po = polib.pofile(path)
except Exception, e:
self.log.error(str(e))
sys.exit(1)
return po
class OptionsUtil:
def __init__(self, options):
self.command_options = options
self.project_config = {}
self.log = Logger()
self.config = ZanataConfig()
def apply_configfiles(self):
url = self.apply_project_config()
username, apikey = self.apply_user_config(url)
# The value in commandline options will overwrite the value in user-config file
if self.command_options.has_key("user_name"):
username = self.command_options["user_name"][0]["value"]
if self.command_options.has_key("key"):
apikey = self.command_options["key"][0]["value"]
self.log.info("zanata server: %s" % url)
return url, username, apikey
def apply_project_config(self):
# Read the project configuration file using --project-config option
config_file = [os.path.join(os.getcwd(), filename) for filename in ["zanata.xml", "flies.xml"]]
if self.command_options.has_key("project_config"):
config_file.append(self.command_options["project_config"][0]["value"])
for path in config_file:
if os.path.exists(path):
self.log.info("Loading zanata project config from: %s" % path)
self.project_config = self.config.read_project_config(path)
break
if not self.project_config:
self.log.info("Can not find zanata.xml, please specify the path of zanata.xml")
# process the url of server
if self.project_config.has_key("project_url"):
url = self.project_config["project_url"]
# The value in options will override the value in project-config file
if self.command_options.has_key("url"):
self.log.info("Overriding url of server with command line option")
url = self.command_options["url"][0]["value"]
if not url or url.isspace():
self.log.error("Please specify valid server url in zanata.xml or with '--url' option")
sys.exit(1)
url = self.trim_url(url)
return url
def trim_url(self, url):
if " " in url or "\n" in url:
self.log.info("Warning, the url which contains '\\n' or whitespace is not valid, please check zanata.xml")
url = url.strip()
if url[-1] == "/":
url = url[:-1]
return url
def get_localemap(self):
if self.project_config and self.project_config.has_key("locale_map"):
locale_map = self.project_config["locale_map"]
return locale_map
def apply_user_config(self, url):
user_name = ""
apikey = ""
# Try to read user-config file
user_config = [
os.path.join(os.path.expanduser("~") + "/.config", filename) for filename in ["zanata.ini", "flies.ini"]
]
if self.command_options.has_key("user_config"):
user_config.append(self.command_options["user_config"][0]["value"])
for path in user_config:
if os.path.exists(path):
self.log.info("Loading zanata user config from: %s" % path)
# Read the user-config file
self.config.set_userconfig(path)
try:
server = self.config.get_server(url)
if server:
user_name = self.config.get_config_value("username", "servers", server)
apikey = self.config.get_config_value("key", "servers", server)
except Exception, e:
self.log.info("Processing user-config file:%s" % str(e))
break
break
if not (user_name, apikey):
self.log.info("Can not find user-config file in home folder, current path or path in 'user-config' option")
return (user_name, apikey)
class CSVConverter:
def __init__(self):
self.log = Logger()
def read_data(self, csv_file):
data = []
try:
reader = csv.reader(open(csv_file, 'rb'))
header = reader.next()
size = len(header)
for line in reader:
items = {}
entry_size = len(line)
for x in range(size):
if x < entry_size:
item = {header[x]: line[x]}
else:
item = {header[x]: ""}
items.update(item)
data.append(items)
except IOError:
self.log.error("Can not find csv file: %s" % csv_file)
return data
def read_csv_file(self, csv_file):
data = []
try:
reader = csv.reader(open(csv_file, 'rb'))
data = [line for line in reader]
except IOError:
self.log.error("Can not find csv file: %s" % csv_file)
return data
def convert_to_json(self, filepath, locale_map, comments_header):
data = self.read_csv_file(expanduser(filepath))
srclocales = []
# srclocales.append('en-US')
entries = []
targetlocales = []
csv_locales = []
comments = []
for index, item in enumerate(data):
terms = []
if index == 0:
# Assuming last two names refers to column names,for example consider following csv file
# en-US,es,ko,ru,pos,description
# Hello,Hola,test,111,noun,Greeting
# first line always contains locales and last two specifies column names
comments = [comm for comm in item[-2:]]
csv_locales = [lc for lc in item[:-2]]
continue
else:
glossary_len = len(item)
csv_locales_len = len(csv_locales)
comments_len = len(comments)
if glossary_len != csv_locales_len + comments_len:
print "Wrong entries in csv file, please check your csv file"
print "Entry in csv file", item
sys.exit(1)
glossary_comments = item[-2:]
for j in range(csv_locales_len):
if j == 0:
term = {'locale': csv_locales[j], 'content': item[j], 'comments': glossary_comments}
else:
term = {'locale': csv_locales[j], 'content': item[j], 'comments': []}
terms.append(term)
entry = {'srcLang': 'en-US', 'glossaryTerms': terms}
entries.append(entry)
glossary = {'sourceLocales': srclocales, 'glossaryEntries': entries, 'targetLocales': targetlocales}
# glossary = {'source-locales':srclocales, 'glossary-entries':entries, 'target-locales':targetlocales}
return json.dumps(glossary)
class ZanataCommand:
def __init__(self, url, http_headers):
self.log = Logger()
self.zanata_resource = ZanataResource(url, http_headers)
def disable_ssl_cert_validation(self):
self.zanata_resource.disable_ssl_cert_validation()
##############################################
#
# Commands for interaction with zanata server
#
##############################################
def get_file_list(self, projectid, iterationid):
return self.zanata_resource.documents.get_file_list(projectid, iterationid)
def get_server_version(self, url):
try:
content = self.zanata_resource.version.get_server_version()
if content:
server_version = content['versionNo']
return server_version
except UnAvaliableResourceException:
self.log.error("Can not retrieve the server version, server may not support the version service")
except UnavailableServiceError:
self.log.error("Service Temporarily Unavailable, stop processing!")
sys.exit(1)
"""
def check_project(self, command_options, project_config):
project_id = ''
iteration_id = ''
if command_options.has_key('project_id'):
project_id = command_options['project_id'][0]['value']
else:
if project_config.has_key('project_id'):
project_id = project_config['project_id']
if command_options.has_key('project_version'):
iteration_id = command_options['project_version'][0]['value']
else:
if project_config.has_key('project_version'):
iteration_id = project_config['project_version']
if not project_id:
self.log.error("Please specify a valid project id in zanata.xml or with '--project-id' option")
sys.exit(1)
if not iteration_id:
self.log.error("Please specify a valid version id in zanata.xml or with '--project-version' option")
sys.exit(1)
self.log.info("Project: %s"%project_id)
self.log.info("Version: %s"%iteration_id)
self.verify_project(project_id, iteration_id)
"""
def verify_project(self, project_id, version_id):
try:
self.zanata_resource.projects.get(project_id)
except NoSuchProjectException, e:
self.log.error(str(e))
sys.exit(1)
try:
self.zanata_resource.projects.iterations.get(project_id, version_id)
except NoSuchProjectException, e:
self.log.error(str(e))
sys.exit(1)
tmlfolder = self.process_srcdir_withsub(self.command_options)
if self.args:
try:
full_path = self.search_file(tmlfolder, self.args[0])
filelist.append(full_path)
except NoSuchFileException, e:
log.error(e.msg)
sys.exit(1)
else:
#get all the pot files from the template folder
publicanutil = PublicanUtility()
filelist = publicanutil.get_file_list(tmlfolder, ".pot")
if not filelist:
log.error("The template folder is empty")
sys.exit(1)
deletefiles = True
return project_type, deletefiles, tmlfolder, filelist
def read_project_config(self, command_options):
project_config = {}
config = ZanataConfig()
#Read the project configuration file using --project-config option
config_file = [os.path.join(os.getcwd(), filename) for filename\
in ['zanata.xml', 'flies.xml']]
if command_options.has_key('project_config'):
config_file.append(command_options['project_config'][0]['value'])
for path in config_file:
class OptionsUtil:
def __init__(self, options):
self.command_options = options
self.project_config = {}
self.log = Logger()
self.config = ZanataConfig()
def apply_configfiles(self):
url = self.apply_project_config()
username, apikey = self.apply_user_config(url)
# The value in commandline options will overwrite the value in user-config file
if self.command_options.has_key('user_name'):
username = self.command_options['user_name'][0]['value']
if self.command_options.has_key('key'):
apikey = self.command_options['key'][0]['value']
self.log.info("zanata server: %s" % url)
return url, username, apikey
def apply_project_config(self):
url = ""
# Read the project configuration file using --project-config option
config_file = [os.path.join(os.getcwd(), filename) for filename
in ['zanata.xml', 'flies.xml']]
if self.command_options.has_key('project_config'):
config_file.append(self.command_options['project_config'][0]['value'])
for path in config_file:
if os.path.exists(path):
self.log.info("Loading zanata project config from: %s" % path)
self.project_config = self.config.read_project_config(path)
break
if not self.project_config:
self.log.info("Can not find zanata.xml, please specify the path of zanata.xml")
# process the url of server
if self.project_config.has_key('project_url'):
url = self.project_config['project_url']
# The value in options will override the value in project-config file
if self.command_options.has_key('url'):
self.log.info("Overriding url of server with command line option")
url = self.command_options['url'][0]['value']
if not url or url.isspace():
self.log.error("Please specify valid server url in zanata.xml or with '--url' option")
sys.exit(1)
url = self.trim_url(url)
return url
def trim_url(self, url):
if ' ' in url or '\n' in url:
self.log.info("Warning, the url which contains '\\n' or whitespace is not valid, please check zanata.xml")
url = url.strip()
if url[-1] == "/":
url = url[:-1]
return url
def get_localemap(self):
if self.project_config and self.project_config.has_key('locale_map'):
locale_map = self.project_config['locale_map']
return locale_map
def apply_user_config(self, url):
user_name = ""
apikey = ""
# Try to read user-config file
user_config = [os.path.join(os.path.expanduser("~") + '/.config', filename) for filename in ['zanata.ini', 'flies.ini']]
if self.command_options.has_key('user_config'):
user_config.append(self.command_options['user_config'][0]['value'])
for path in user_config:
if os.path.exists(path):
self.log.info("Loading zanata user config from: %s" % path)
# Read the user-config file
self.config.set_userconfig(path)
try:
server = self.config.get_server(url)
if server:
user_name = self.config.get_config_value("username", "servers", server)
apikey = self.config.get_config_value("key", "servers", server)
except Exception, e:
self.log.info("Processing user-config file:%s" % str(e))
break
break
if not (user_name, apikey):
self.log.info("Can not find user-config file in home folder, current path or path in 'user-config' option")
return (user_name, apikey)
class PublicanUtility:
def __init__(self):
self.log = Logger()
def create_txtflow(self, pofile):
"""
Convert the content of the pot file to a list of text flow.
@return: the dictionary object of textflow
"""
textflows = []
for entry in pofile:
context = None
reflist = []
content = ""
if entry.msgctxt is not None:
hashbase = entry.msgctxt + u"\u0000" + entry.msgid
context = entry.msgctxt
else:
hashbase = entry.msgid
# pylint: disable=E1101
m = hashlib.md5()
m.update(hashbase.encode('utf-8'))
textflowId = m.hexdigest()
"""
"extensions":[{"object-type":"pot-entry-header","context":"context",
"references":["fff"],"extractedComment":"extractedComment",
"flags":["java-format"]}]
"""
extracted_comment = entry.comment
references = entry.occurrences
for ref in references:
node = ref[0] + ":" + ref[1]
reflist.append(node)
flags = entry.flags
if entry.msgid_plural:
content = [entry.msgid, entry.msgid_plural]
else:
content = entry.msgid
if context is not None:
extensions = [{
'object-type': 'comment',
'value': extracted_comment,
'space': 'preserve'
}, {
"object-type": "pot-entry-header",
"context": context,
"references": reflist,
"extractedComment": '',
"flags": flags
}]
else:
extensions = [{
'object-type': 'comment',
'value': extracted_comment,
'space': 'preserve'
}, {
"object-type": "pot-entry-header",
"references": reflist,
"extractedComment": '',
"flags": flags
}]
if entry.msgid_plural:
textflow = {
'id': textflowId,
'lang': 'en-US',
'contents': content,
'plural': 'true',
'extensions': extensions
}
else:
textflow = {
'id': textflowId,
'lang': 'en-US',
'content': content,
'plural': 'false',
'extensions': extensions
}
textflows.append(textflow)
return textflows
def check_empty(self, contents):
for string in contents:
if string != u'':
return False
return True
def check_nonempty(self, contents):
for string in contents:
if string == u'':
return False
return True
def get_contentstate(self, entry):
"""
Determine the ContentState for a PO entry, based on contents and fuzzy flag
@return: the state
"""
fuzzy = False
contents = []
if "fuzzy" in entry.flags:
fuzzy = True
if entry.msgid_plural:
keys = entry.msgstr_plural.keys()
keys.sort()
for key in keys:
contents.append(entry.msgstr_plural[key])
else:
contents.append(entry.msgstr)
if self.check_empty(contents):
fuzzy = False
if fuzzy:
return "NeedReview"
if self.check_nonempty(contents):
return "Approved"
else:
return "New"
def create_txtflowtarget(self, pofile):
"""
Convert the content of the po file to a list of textflowtarget.
@return: the dictionary object of textflow
"""
obs_list = pofile.obsolete_entries()
textflowtargets = []
content = ""
for entry in pofile:
if entry in obs_list:
continue
if entry.msgctxt is not None:
hashbase = entry.msgctxt + u"\u0000" + entry.msgid
else:
hashbase = entry.msgid
# pylint: disable=E1101
m = hashlib.md5()
m.update(hashbase.encode('utf-8'))
textflowId = m.hexdigest()
translator_comment = entry.tcomment
state = self.get_contentstate(entry)
#create extensions
extensions = [{
"object-type": "comment",
"value": translator_comment,
"space": "preserve"
}]
if entry.msgid_plural:
content = []
keys = entry.msgstr_plural.keys()
keys.sort()
for key in keys:
content.append(entry.msgstr_plural[key])
textflowtarget = {
'resId': textflowId,
'state': state,
'contents': content,
'extensions': extensions
}
else:
content = entry.msgstr
textflowtarget = {
'resId': textflowId,
'state': state,
'content': content,
'extensions': extensions
}
textflowtargets.append(textflowtarget)
return textflowtargets
def validate_content_type(self, content_type, object_type):
PATTERN = r'.+? charset=([\w_\-:\.]+)'
rxt = re.compile(unicode(PATTERN))
match = rxt.search(content_type)
if match:
enc = match.group(1).strip()
if enc not in ["UTF-8", "utf-8", "utf8", "ascii", "UTF8", "ASCII"]:
if enc == 'CHARSET':
if object_type == 'po-target-header':
self.log.error(
"Invalid encoding CHARSET; please correct the Content-Type charset (UTF-8 recommended)"
)
sys.exit(1)
else:
self.log.error(
"Unsupported encoding; please change the Content-Type charset (UTF-8 recommended)"
)
sys.exit(1)
def create_extensions(self, pofile, object_type):
"""
"extensions":[{"object-type":"po-header","comment":"comment_value", "entries":[{"key":"h1","value":"v1"}]}]
"extensions":[{"object-type":"po-target-header", "comment":"comment_value", "entries":[{"key":"ht","value":"vt1"}]}]
"""
entries = []
metadatas = pofile.ordered_metadata()
for item in metadatas:
entry = {"key": item[0], "value": item[1]}
entries.append(entry)
if pofile.metadata.has_key('Content-Type'):
self.validate_content_type(pofile.metadata['Content-Type'],
object_type)
extensions = [{
"object-type": object_type,
"comment": pofile.header,
"entries": entries
}]
return extensions
def create_pofile(self, path):
"""
Convert the po file to a pofile object in polib.
@return: pofile object
"""
try:
po = polib.pofile(path)
except Exception, e:
self.log.error(str(e))
sys.exit(1)
return po
class PublicanUtility:
def __init__(self):
self.log = Logger()
def create_txtflow(self, pofile):
"""
Convert the content of the pot file to a list of text flow.
@return: the dictionary object of textflow
"""
textflows = []
for entry in pofile:
reflist = []
if entry.msgctxt:
self.log.warn("encountered msgctxt; not currently supported")
m = hashlib.md5()
m.update(entry.msgid.encode('utf-8'))
textflowId = m.hexdigest()
"""
"extensions":[{"object-type":"pot-entry-header","context":"context",
"references":["fff"],"extractedComment":"extractedComment",
"flags":["java-format"]}]
"""
extracted_comment = entry.comment
references = entry.occurrences
for ref in references:
node = ref[0]+":"+ref[1]
reflist.append(node)
flags = entry.flags
#extensions_single_comment = [{'object-type':'comment','value':'test','space':'preserve'}]
#extensions_pot_entry_header = [{"object-type":"pot-entry-header","context":"context","references":["fff"],"extractedComment":"extractedComment","flags":["java-format"]}]
extensions=[{'object-type':'comment','value':extracted_comment,'space':'preserve'}, {"object-type":"pot-entry-header","context":"","references":reflist,"extractedComment":'',"flags":flags}]
textflow = {'id': textflowId, 'lang':'en-US', 'content':entry.msgid, 'extensions':extensions, 'revision':1}
textflows.append(textflow)
return textflows
def create_txtflowtarget(self, pofile):
"""
Convert the content of the po file to a list of textflowtarget.
@return: the dictionary object of textflow
"""
obs_list=pofile.obsolete_entries()
textflowtargets = []
for entry in pofile:
if entry in obs_list:
continue
m = hashlib.md5()
m.update(entry.msgid.encode('utf-8'))
textflowId = m.hexdigest()
comment = entry.comment
if entry.msgstr:
state = "Approved"
else:
state = "New"
#need judge for fuzzy state
#create extensions
extensions = [{"object-type":"comment","value":comment,"space":"preserve"}]
# {"resId":"782f49c4e93c32403ba0b51821b38b90","state":"Approved","translator":{"email":"id","name":"name"},"content":"title:
# ttff","extensions":[{"object-type":"comment","value":"testcomment","space":"preserve"}]}
# Diable the translator to avoid issues on server side
textflowtarget = {'resId': textflowId, 'state': state, 'content':entry.msgstr,'extensions':extensions}
#Temporary fill in the admin info for translator to pass the validation, waiting for server side change
textflowtargets.append(textflowtarget)
return textflowtargets
def create_extensions(self, pofile, object_type):
"""
"extensions":[{"object-type":"po-header","comment":"comment_value", "entries":[{"key":"h1","value":"v1"}]}]
"extensions":[{"object-type":"po-target-header", "comment":"comment_value", "entries":[{"key":"ht","value":"vt1"}]}]
"""
entries = []
metadatas = pofile.ordered_metadata()
for item in metadatas:
entry = {"key":item[0], "value":item[1]}
entries.append(entry)
extensions = [{"object-type":object_type,"comment":pofile.header, "entries":entries}]
return extensions
def create_pofile(self, path):
"""
Convert the po file to a pofile object in polib.
@return: pofile object
"""
try:
po = polib.pofile(path)
except Exception:
self.log.error("Can not processing the po file")
sys.exit()
return po
def get_file_list(self, path, file_type):
final_file_list = []
root_list = os.listdir(path)
for item in root_list:
if item == '.svn':
continue
full_path = os.path.join(path, item)
if full_path.endswith(file_type):
final_file_list.append(full_path)
if os.path.isdir(full_path):
final_file_list+=self.get_file_list(full_path, file_type)
return final_file_list
def hash_match(self, message, msgid):
m = hashlib.md5()
m.update(message.msgid.encode('utf-8'))
if m.hexdigest() == msgid:
return True
else:
return False
def strip_path(self, full_path, root_path):
if root_path[-1] != "/":
root_path = root_path+'/'
filename = full_path.split(root_path)[1]
if '.' in filename:
# Strip the file name
filename = filename.split('.')[0]
return filename
def potfile_to_json(self, filepath, root_path):
"""
Parse the pot file, create the request body
@param filepath: the path of the pot file
"""
filename = self.strip_path(filepath, root_path)
pofile = self.create_pofile(filepath)
textflows = self.create_txtflow(pofile)
extensions = self.create_extensions(pofile, "po-header")
items = {'name':filename, 'contentType':'application/x-gettext', 'lang':'en-US', 'extensions':extensions, 'textFlows':textflows}
return json.dumps(items), filename
def pofile_to_json(self, filepath):
"""
Parse the po file, create the request body
@param filepath: the path of the po file
"""
pofile = self.create_pofile(filepath)
textflowtargets = self.create_txtflowtarget(pofile)
#the function for extensions have not implemented yet
extensions = self.create_extensions(pofile, "po-target-header")
items = {'links':[],'extensions':extensions, 'textFlowTargets':textflowtargets}
return json.dumps(items)
def save_to_pofile(self, path, translations, pot):
"""
Save PO file to path, based on json objects of pot and translations
@param translations: the json object of the content retrieved from server
@param path: the po folder for output
@param pot: the json object of the pot retrieved from server
"""
po = polib.POFile(fpath=path)
potcontent = json.loads(pot)
# pylint: disable-msg=E1103
textflows = potcontent.get('textFlows')
if potcontent.get('extensions'):
extensions = potcontent.get('extensions')[0]
po.header = extensions.get('comment')
for item in extensions.get('entries'):
po.metadata[item['key']]=item['value']
else:
raise InvalidPOTFileException("Error", "the extensions of Resource is empty")
for textflow in textflows:
if textflow.get('extensions'):
poentry = polib.POEntry(occurrences=None)
entry_list = textflow.get('extensions')
for entry in entry_list:
if entry.get('object-type') == 'pot-entry-header':
#PotEntryHeader
#Check the references is not empty
if entry.get('references')!=[u'']:
for item in entry.get('references'):
#in some cases, entry contains more than one reference
if ' ' in item:
reference = item.split(' ')
ref_list = []
for i in reference:
ref = tuple(i.split(':'))
ref_list.append(ref)
poentry.occurrences= ref_list
else:
poentry.occurrences = [tuple(item.split(':'))]
else:
poentry.occurrences = None
#print poentry.occurrences
poentry.flags = entry.get('flags')
if entry.get('object-type') == 'comment':
#SimpleComment
poentry.comment = entry.get('value')
poentry.msgid = textflow.get('content')
po.append(poentry)
else:
raise InvalidPOTFileException("Error", "the extensions of TextFlow is empty")
#If the translation is exist, read the content of the po file
if translations:
content = json.loads(translations)
"""
"extensions":[{"object-type":"po-target-header", "comment":"comment_value", "entries":[{"key":"ht","value":"vt1"}]}]
"""
if content.get('extensions'):
ext = content.get('extensions')[0]
header_comment = ext.get('comment')
if header_comment:
po.header = header_comment
for item in ext.get('entries'):
po.metadata[item['key']]=item['value']
targets = content.get('textFlowTargets')
"""
"extensions":[{"object-type":"comment","value":"testcomment","space":"preserve"}]
"""
# copy any other stuff you need to transfer
for message in po:
for translation in targets:
if translation.get('extensions'):
extensions=translation.get('extensions')[0]
#if extensions:
# ext_type = extensions.get('object-type')
# comment = extensions.get('comment')
# entries = extensions.get('value')
if self.hash_match(message, translation.get('resId')):
message.msgstr = translation.get('content')
# finally save resulting po to outpath as lang/myfile.po
po.save()
# pylint: disable-msg=E1103
self.log.info("Writing po file to %s"%(path))
class ZanataCommand:
def __init__(self, url, username=None, apikey=None, http_headers=None):
self.log = Logger()
self.zanata_resource = ZanataResource(url, username, apikey,
http_headers)
def disable_ssl_cert_validation(self):
self.zanata_resource.disable_ssl_cert_validation()
##############################################
#
# Commands for interaction with zanata server
#
##############################################
def get_file_list(self, projectid, iterationid):
return self.zanata_resource.documents.get_file_list(
projectid, iterationid)
def get_server_version(self, url):
try:
content = self.zanata_resource.version.get_server_version()
if content:
server_version = content['versionNo']
return server_version
except UnAvaliableResourceException:
self.log.error(
"Can not retrieve the server version, server may not support the version service"
)
except UnavailableServiceError:
self.log.error("Service Temporarily Unavailable, stop processing!")
sys.exit(1)
"""
def check_project(self, command_options, project_config):
project_id = ''
iteration_id = ''
if command_options.has_key('project_id'):
project_id = command_options['project_id'][0]['value']
else:
if project_config.has_key('project_id'):
project_id = project_config['project_id']
if command_options.has_key('project_version'):
iteration_id = command_options['project_version'][0]['value']
else:
if project_config.has_key('project_version'):
iteration_id = project_config['project_version']
if not project_id:
self.log.error("Please specify a valid project id in zanata.xml or with '--project-id' option")
sys.exit(1)
if not iteration_id:
self.log.error("Please specify a valid version id in zanata.xml or with '--project-version' option")
sys.exit(1)
self.log.info("Project: %s"%project_id)
self.log.info("Version: %s"%iteration_id)
self.verify_project(project_id, iteration_id)
"""
def verify_project(self, project_id, version_id):
try:
self.zanata_resource.projects.get(project_id)
except NoSuchProjectException, e:
self.log.error(str(e))
sys.exit(1)
try:
self.zanata_resource.projects.iterations.get(
project_id, version_id)
except NoSuchProjectException, e:
self.log.error(str(e))
sys.exit(1)
