def initialize():
loadConfig()
keyfile = None
if config.has_option('api', 'service_key'):
keyfile = os.path.expanduser(config.get('api', 'service_key'))
if not os.path.exists(keyfile):
warning("Could not find service key JSON at " + keyfile)
keyfile = None
global client, parent, project_id
project_id = config.get('api', 'project_id').strip()
if project_id == "":
critical("Translation requires Project ID to be set in " + confPath)
else:
try:
if keyfile is not None:
client = api.TranslationServiceClient.from_service_account_file(
keyfile)
else:
client = api.TranslationServiceClient()
parent = client.location_path(project_id, 'global')
except Exception as err:
critical(err)
def translate():
from google.cloud import translate_v3beta1 as translate
client = translate.TranslationServiceClient()
project_id = 'data-onboarding-ez'
location = 'global'
parent = client.location_path(project_id, location)
# The text to translate
list = [
u'<span style=color: rgb(255,0,0);>・削除でお願いします。</span>',
u'<strong>質疑応答の内容（後日の回答分を含む）、はこちら！</strong>',
u'<strong>SDXCカードを初めて差したときには「本体更新」を求められます。</strong>'
]
length = len(list)
# The target language
source = 'ja'
target = 'en'
# Loop for translation
# Translates some text into Japanese
for i in range(length):
response = client.translate_text(parent=parent,
contents=[list[i]],
mime_type='text/plain',
source_language_code=source,
target_language_code=target)
print('String' + str(i + 1) + ':')
print(list[i])
for translation in response.translations:
print('{}'.format(translation))
print(' ')
def test_get_supported_languages(self):
# Setup Expected Response
expected_response = {}
expected_response = translation_service_pb2.SupportedLanguages(
**expected_response
)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = translate_v3beta1.TranslationServiceClient()
# Setup Request
parent = client.location_path("[PROJECT]", "[LOCATION]")
response = client.get_supported_languages(parent)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = translation_service_pb2.GetSupportedLanguagesRequest(
parent=parent
)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def translate(text, source_language, target_language):
from google.cloud import translate_v3beta1 as translate
client = translate.TranslationServiceClient()
project_id = "lunch-scraper"
location = "global"
parent = client.location_path(project_id, location)
if os.environ.get('CI') != None:
return text
if isinstance(text, list):
text = "\n".join(text)
response = client.translate_text(
parent=parent,
contents=[text],
mime_type='text/plain', # mime types: text/plain, text/html
source_language_code=source_language,
target_language_code=target_language)
try:
response = response.translations[0].translated_text
response = response.split("\n")
finally:
return response
def translate_document(project_id: str, file_path: str):
client = translate.TranslationServiceClient()
location = "us-central1"
parent = f"projects/{project_id}/locations/{location}"
# Supported file types: https://cloud.google.com/translate/docs/supported-formats
with open(file_path, "rb") as document:
document_content = document.read()
document_input_config = {
"content": document_content,
"mime_type": "application/pdf",
}
response = client.translate_document(
request={
"parent": parent,
"target_language_code": "fr-FR",
"document_input_config": document_input_config,
})
# To view translated document, write `response.document_translation.byte_stream_outputs` to file.
# If not provided in the TranslationRequest, the translated file will only be returned through a byte-stream
# and its output mime type will be the same as the input file's mime type
print("Response: Detected Language Code - {}".format(
response.document_translation.detected_language_code))
def test_translate_text(self):
# Setup Expected Response
expected_response = {}
expected_response = translation_service_pb2.TranslateTextResponse(
**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = translate_v3beta1.TranslationServiceClient()
# Setup Request
contents = []
target_language_code = "targetLanguageCode1323228230"
parent = client.location_path("[PROJECT]", "[LOCATION]")
response = client.translate_text(contents, target_language_code,
parent)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = translation_service_pb2.TranslateTextRequest(
contents=contents,
target_language_code=target_language_code,
parent=parent)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_delete_glossary(self):
# Setup Expected Response
name_2 = "name2-1052831874"
expected_response = {"name": name_2}
expected_response = translation_service_pb2.DeleteGlossaryResponse(
**expected_response)
operation = operations_pb2.Operation(
name="operations/test_delete_glossary", done=True)
operation.response.Pack(expected_response)
# Mock the API response
channel = ChannelStub(responses=[operation])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = translate_v3beta1.TranslationServiceClient()
# Setup Request
name = client.glossary_path("[PROJECT]", "[LOCATION]", "[GLOSSARY]")
response = client.delete_glossary(name)
result = response.result()
assert expected_response == result
assert len(channel.requests) == 1
expected_request = translation_service_pb2.DeleteGlossaryRequest(
name=name)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def google_translate(questions, output):
os.environ[
'GOOGLE_APPLICATION_CREDENTIALS'] = '/Users/peixuan/Downloads/green-chalice-237310-5fa1959eb742.json'
client = translate.TranslationServiceClient()
project_id = 'green-chalice-237310'
location = 'us-central1'
parent = client.location_path(project_id, location)
with open(output, 'w') as fout:
while len(questions) > 0:
subset_questions = questions[:5]
questions = questions[5:]
cn_response = client.translate_text(parent=parent,
contents=subset_questions,
source_language_code='en',
target_language_code='zh-CN')
cn_questions = []
for cn_question in cn_response.translations:
cn_questions.append(cn_question.translated_text)
# print('cn: {}'.format(cn_response.translations[0].translated_text))
# print(cn_questions)
en_response = client.translate_text(parent=parent,
contents=cn_questions,
mime_type='text/plain',
source_language_code='zh-CN',
target_language_code='en')
# print('en: {}'.format(en_response.translations[0].translated_text))
for en_question in en_response.translations:
# print(en_question.translated_text)
fout.write(en_question.translated_text + '\n')
def test_create_glossary(self):
# Setup Expected Response
name = "name3373707"
entry_count = 811131134
expected_response = {"name": name, "entry_count": entry_count}
expected_response = translation_service_pb2.Glossary(
**expected_response)
operation = operations_pb2.Operation(
name="operations/test_create_glossary", done=True)
operation.response.Pack(expected_response)
# Mock the API response
channel = ChannelStub(responses=[operation])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = translate_v3beta1.TranslationServiceClient()
# Setup Request
parent = client.location_path("[PROJECT]", "[LOCATION]")
glossary = {}
response = client.create_glossary(parent, glossary)
result = response.result()
assert expected_response == result
assert len(channel.requests) == 1
expected_request = translation_service_pb2.CreateGlossaryRequest(
parent=parent, glossary=glossary)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def create_glossary(project_id, glossary_id):
# [START translate_create_glossary_beta]
from google.cloud import translate_v3beta1 as translate
client = translate.TranslationServiceClient()
# project_id = 'YOUR_PROJECT_ID'
# glossary_id = 'glossary-id'
location = 'us-central1' # The location of the glossary
name = client.glossary_path(project_id, location, glossary_id)
language_codes_set = translate.types.Glossary.LanguageCodesSet(
language_codes=['en', 'es'])
gcs_source = translate.types.GcsSource(
input_uri='gs://cloud-samples-data/translation/glossary.csv')
input_config = translate.types.GlossaryInputConfig(gcs_source=gcs_source)
glossary = translate.types.Glossary(name=name,
language_codes_set=language_codes_set,
input_config=input_config)
parent = client.location_path(project_id, location)
operation = client.create_glossary(parent=parent, glossary=glossary)
result = operation.result(timeout=90)
print(u'Created: {}'.format(result.name))
print(u'Input Uri: {}'.format(result.input_config.gcs_source.input_uri))
def batch_translate_text(project_id, input_uri, output_uri):
# [START translate_batch_translate_text_beta]
from google.cloud import translate_v3beta1 as translate
client = translate.TranslationServiceClient()
# project_id = YOUR_PROJECT_ID
# input_uri = 'gs://cloud-samples-data/translation/text.txt'
# output_uri = 'gs://YOUR_BUCKET_ID/path_to_store_results/'
location = 'us-central1'
parent = client.location_path(project_id, location)
gcs_source = translate.types.GcsSource(input_uri=input_uri)
input_config = translate.types.InputConfig(
mime_type='text/plain', # mime types: text/plain, text/html
gcs_source=gcs_source)
gcs_destination = translate.types.GcsDestination(
output_uri_prefix=output_uri)
output_config = translate.types.OutputConfig(
gcs_destination=gcs_destination)
operation = client.batch_translate_text(parent=parent,
source_language_code='en-US',
target_language_codes=['sr-Latn'],
input_configs=[input_config],
output_config=output_config)
result = operation.result(timeout=240)
print(u'Total Characters: {}'.format(result.total_characters))
print(u'Translated Characters: {}'.format(result.translated_characters))
def tran_v3(content, model, syst):
# model = model[2:]
if model == 'pbmt':
model = 'base'
else:
model = 'nmt'
basepath = os.path.abspath('.')
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = os.path.join(
basepath, 'Ipsos-translation-c094ada851a7.json')
client = translate.TranslationServiceClient()
project_id = "ipsos-translation"
location = 'global'
parent = client.location_path(project_id, location)
response = client.translate_text(
parent=parent,
contents=[content],
mime_type='text/plain', # mime types: text/plain, text/html
model=
'projects/ipsos-translation/locations/global/models/general/%s' %
model,
source_language_code='en-US',
target_language_code='zh-CN')
result = response.translations[0].translated_text
return result
def main():
# Instantiates a client
client = translate.TranslationServiceClient()
project_id = "hello-everyone-240322"
location = "global"
parent = client.location_path(project_id, location)
# The text to translate
text = "Hello, everyone"
# The target language
target = choice(client.get_supported_languages(parent).languages).language_code
# Translation
resp = client.translate_text(
parent=parent,
contents=[text],
mime_type='text/plain', # mime types: text/plain, text/html
source_language_code='en-US',
target_language_code=target
)
for translation in resp.translations:
print(f"'{target}' -> '{translation.translated_text}'")
def translate_text_with_glossary(project_id, glossary_id, text):
# [START translate_translate_text_with_glossary_beta]
from google.cloud import translate_v3beta1 as translate
client = translate.TranslationServiceClient()
# project_id = 'YOUR_PROJECT_ID'
# glossary_id = 'GLOSSARY_ID'
# text = 'Text you wish to translate'
location = 'us-central1' # The location of the glossary
glossary = client.glossary_path(
project_id,
'us-central1', # The location of the glossary
glossary_id)
glossary_config = translate.types.TranslateTextGlossaryConfig(
glossary=glossary)
parent = client.location_path(project_id, location)
result = client.translate_text(
parent=parent,
contents=[text],
mime_type='text/plain', # mime types: text/plain, text/html
source_language_code='en',
target_language_code='es',
glossary_config=glossary_config)
for translation in result.glossary_translations:
print(translation)
def translate_text_with_glossary(project_id, glossary_id, text):
# [START translate_translate_text_with_glossary_beta]
from google.cloud import translate_v3beta1 as translate
client = translate.TranslationServiceClient()
# project_id = 'YOUR_PROJECT_ID'
# glossary_id = 'GLOSSARY_ID'
# text = 'Text you wish to translate'
location = 'us-central1' # The location of the glossary
glossary = client.glossary_path(
project_id,
'us-central1', # The location of the glossary
glossary_id)
glossary_config = translate.types.TranslateTextGlossaryConfig(
glossary=glossary)
parent = f"projects/{project_id}/locations/{location}"
result = client.translate_text(
request={
"parent": parent,
"contents": [text],
"mime_type": "text/plain",
"source_language_code": "en",
"target_language_code": "es",
"glossary_config": glossary_config
}
)
for translation in result.glossary_translations:
print(translation)
def test_list_glossaries(self):
# Setup Expected Response
next_page_token = ""
glossaries_element = {}
glossaries = [glossaries_element]
expected_response = {
"next_page_token": next_page_token,
"glossaries": glossaries,
}
expected_response = translation_service_pb2.ListGlossariesResponse(
**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = translate_v3beta1.TranslationServiceClient()
# Setup Request
parent = client.location_path("[PROJECT]", "[LOCATION]")
paged_list_response = client.list_glossaries(parent)
resources = list(paged_list_response)
assert len(resources) == 1
assert expected_response.glossaries[0] == resources[0]
assert len(channel.requests) == 1
expected_request = translation_service_pb2.ListGlossariesRequest(
parent=parent)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def translate_this(
text,
source_language='en-us',
target_language='es-ES',
location='global',
project_id='leo-ios-244300',
GOOGLE_APPLICATION_CREDENTIALS="/Users/adam/Code/leo-web/leo_translate/leo-ios-244300-ca1482078257.json"
):
os.environ[
'GOOGLE_APPLICATION_CREDENTIALS'] = GOOGLE_APPLICATION_CREDENTIALS
client = translate.TranslationServiceClient()
parent = client.location_path(project_id, location)
response = client.translate_text(
parent=parent,
contents=text,
mime_type='text/plain', # mime types: text/plain, text/html
source_language_code=source_language,
target_language_code=target_language)
translations = []
for translation in response.translations:
translations.append(translation.translated_text)
print('Translated this:', translations)
if source_language == 'en-us':
return translations[0]
if source_language == 'es-ES':
return translations
def test_batch_translate_text_exception(self):
# Setup Response
error = status_pb2.Status()
operation = operations_pb2.Operation(
name="operations/test_batch_translate_text_exception", done=True)
operation.error.CopyFrom(error)
# Mock the API response
channel = ChannelStub(responses=[operation])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = translate_v3beta1.TranslationServiceClient()
# Setup Request
parent = client.location_path("[PROJECT]", "[LOCATION]")
source_language_code = "sourceLanguageCode1687263568"
target_language_codes = []
input_configs = []
output_config = {}
response = client.batch_translate_text(
parent,
source_language_code,
target_language_codes,
input_configs,
output_config,
)
exception = response.exception()
assert exception.errors[0] == error
def test_get_glossary(self):
# Setup Expected Response
name_2 = "name2-1052831874"
entry_count = 811131134
expected_response = {"name": name_2, "entry_count": entry_count}
expected_response = translation_service_pb2.Glossary(
**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = translate_v3beta1.TranslationServiceClient()
# Setup Request
name = client.glossary_path("[PROJECT]", "[LOCATION]", "[GLOSSARY]")
response = client.get_glossary(name)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = translation_service_pb2.GetGlossaryRequest(
name=name)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def run(self):
logger.info('Now moving on to translate columns')
location = 'global'
client = translate.TranslationServiceClient()
parent = client.location_path(project_id, location)
if 'raw_scrape_df' not in ctx:
translation = pd.read_csv(
str(ctx['sysFolder']) + '/run/Raw Scrape.csv')
else:
translation = ctx['raw_scrape_df']
for i in range(8):
try:
translation.columns = [
client.translate_text(parent=parent,
contents=[x],
target_language_code="en",
source_language_code="ja").
translations[0].translated_text
for x in translation.columns
]
break
except:
logger.warning('Translation failed. Warning: {0}'.format(
str(sys.exc_info())))
logger.info('Attempt: {0}'.format(str(i)))
logger.info('Rest for 60 seconds and retry')
time.sleep(60)
logger.info('Translation completed successfully')
##### Cleaning and Restructuring of Data #####
logger.info('Writing Final Output File')
try:
the_cols = [
'Office Name', 'Rental Link', 'Scrapped Date',
'Street address', 'Latitude', 'Longtitude', 'Monthly cost',
'Tsubo unit price (rent) (common expenses)', 'Basis number',
'Brokerage fee', 'Security deposit', 'Amortization',
'key money', 'Available days', 'Tsuba unit price', 'Use',
'renewal', 'Rank', 'Completion', 'Application / specification',
'Nearest station', 'Standard floor plan number',
'Earthquake resistance', 'Elevator', 'entrance',
'air conditioning', 'Floor specification', 'Ceiling height',
'toilet', 'Parking facility', 'Neighboring Area / Station JP',
'simPropAdd', 'simPropArea', 'simPropName', 'simPropPrice',
'Review Description', 'Review Rating', 'Tenant Link',
'Tenant Name'
]
dfdata = {}
for x in the_cols:
dfdata[x] = translation[x]
translation = pd.DataFrame(dfdata)
except:
logger.warning(
'Restructuring fail as some fields are not present. Not going to restructure.'
)
translation.to_csv(self.output().path)
def test_list_glossaries_exception(self):
channel = ChannelStub(responses=[CustomException()])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = translate_v3beta1.TranslationServiceClient()
paged_list_response = client.list_glossaries()
with pytest.raises(CustomException):
list(paged_list_response)
def test_get_supported_languages_exception(self):
# Mock the API response
channel = ChannelStub(responses=[CustomException()])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = translate_v3beta1.TranslationServiceClient()
with pytest.raises(CustomException):
client.get_supported_languages()
def create_glossary(languages, project_id, glossary_name, glossary_uri):
"""Creates a GCP glossary resource
Assumes you've already manually uploaded a glossary to Cloud Storage
ARGS
languages: list of languages in the glossary
project_id: GCP project id
glossary_name: name you want to give this glossary resource
glossary_uri: the uri of the glossary you uploaded to Cloud Storage
RETURNS
nothing
"""
# Instantiates a client
client = translate.TranslationServiceClient()
# Designates the data center location that you want to use
location = 'us-central1'
# Set glossary resource name
name = client.glossary_path(
project_id,
location,
glossary_name)
# Set language codes
language_codes_set = translate.types.Glossary.LanguageCodesSet(
language_codes=languages)
gcs_source = translate.types.GcsSource(
input_uri=glossary_uri)
input_config = translate.types.GlossaryInputConfig(
gcs_source=gcs_source)
# Set glossary resource information
glossary = translate.types.Glossary(
name=name,
language_codes_set=language_codes_set,
input_config=input_config)
parent = client.location_path(project_id, location)
# Create glossary resource
# Handle exception for case in which a glossary
# with glossary_name already exists
try:
operation = client.create_glossary(parent=parent, glossary=glossary)
operation.result(timeout=90)
print('Created glossary ' + glossary_name + '.')
except AlreadyExists:
print('The glossary ' + glossary_name +
' already exists. No new glossary was created.')
def __init__(self, *args, **kwargs):
from google.cloud import translate_v3beta1 as google_translator
# don't forget to set $GOOGLE_APPLICATION_CREDENTIALS to the json
# with the authentication data
self.client = google_translator.TranslationServiceClient()
self.project_id = os.getenv('GOOGLE_PROJECT_ID', '')
self.location = 'global'
self.parent = self.client.location_path(self.project_id, self.location)
super().__init__(*args, **kwargs)
def test_batch_translate_text(self):
# Setup Expected Response
total_characters = 1368640955
translated_characters = 1337326221
failed_characters = 1723028396
expected_response = {
"total_characters": total_characters,
"translated_characters": translated_characters,
"failed_characters": failed_characters,
}
expected_response = translation_service_pb2.BatchTranslateResponse(
**expected_response
)
operation = operations_pb2.Operation(
name="operations/test_batch_translate_text", done=True
)
operation.response.Pack(expected_response)
# Mock the API response
channel = ChannelStub(responses=[operation])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = translate_v3beta1.TranslationServiceClient()
# Setup Request
parent = client.location_path("[PROJECT]", "[LOCATION]")
source_language_code = "sourceLanguageCode1687263568"
target_language_codes = []
input_configs = []
output_config = {}
response = client.batch_translate_text(
parent,
source_language_code,
target_language_codes,
input_configs,
output_config,
)
result = response.result()
assert expected_response == result
assert len(channel.requests) == 1
expected_request = translation_service_pb2.BatchTranslateTextRequest(
parent=parent,
source_language_code=source_language_code,
target_language_codes=target_language_codes,
input_configs=input_configs,
output_config=output_config,
)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_get_glossary_exception(self):
# Mock the API response
channel = ChannelStub(responses=[CustomException()])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = translate_v3beta1.TranslationServiceClient()
# Setup request
name = client.glossary_path("[PROJECT]", "[LOCATION]", "[GLOSSARY]")
with pytest.raises(CustomException):
client.get_glossary(name)
def setup(self):
self._client = translate_v3beta1.TranslationServiceClient()
self._parent_inside = self._client.location_path(
PROJECT_INSIDE, "us-central1")
self._parent_outside = self._client.location_path(
PROJECT_OUTSIDE, "us-central1")
def make_glossary_name(project_id):
return "projects/{0}/locations/us-central1/glossaries/fake_glossary".format(
project_id)
self._glossary_name_inside = make_glossary_name(PROJECT_INSIDE)
self._glossary_name_outside = make_glossary_name(PROJECT_OUTSIDE)
def test_detect_language_exception(self):
# Mock the API response
channel = ChannelStub(responses=[CustomException()])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = translate_v3beta1.TranslationServiceClient()
# Setup request
parent = client.location_path("[PROJECT]", "[LOCATION]")
with pytest.raises(CustomException):
client.detect_language(parent)
def test_translate_text_exception(self):
# Mock the API response
channel = ChannelStub(responses=[CustomException()])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = translate_v3beta1.TranslationServiceClient()
# Setup request
contents = []
target_language_code = "targetLanguageCode1323228230"
with pytest.raises(CustomException):
client.translate_text(contents, target_language_code)
def translate_text(self, text, source_language_code, target_language_code,
project_id):
"""Translates text to a given language using a glossary
ARGS
text: String of text to translate
source_language_code: language of input text
target_language_code: language of output text
project_id: GCP project id
glossary_name: name you gave your project's glossary
resource when you created it
RETURNS
String of translated text
"""
# Instantiates a client
client = translate.TranslationServiceClient()
# Designates the data center location that you want to use
location = 'us-central1'
parent = client.location_path(project_id, location)
result = client.translate_text(
parent=parent,
contents=[text],
mime_type='text/plain', # mime types: text/plain, text/html
source_language_code=source_language_code,
target_language_code=target_language_code)
print(result)
# Extract translated text from API response
return result
def ipa(self):
ipa_list = []
for word in translated:
ipa_list.append(symbol)
def data(self):
data = {
"Image" : self.img,
"OriginalText" : text_to_translate,
"TranslatedText" : translated,
"IPA" : translated.ipa()
}
result = firebase.post('https://qhacks2020-86c07.firebaseio.com')