def test_add_file_to_zip(self):
self.tmp_dir1 = tempfile.mkdtemp(prefix='Door43_test_file_utils_')
zip_file = os.path.join(self.tmp_dir1, 'foo.zip')
_, self.tmp_file = tempfile.mkstemp(prefix='Door43_test_')
with open(self.tmp_file, "w") as tmpf:
tmpf.write("hello world")
with zipfile.ZipFile(zip_file, "w"):
pass # create empty archive
file_utils.add_file_to_zip(zip_file, self.tmp_file,
os.path.basename(self.tmp_file))
with zipfile.ZipFile(zip_file, "r") as zf:
with zf.open(os.path.basename(self.tmp_file), "r") as f:
self.assertEqual(f.read().decode("ascii"), "hello world")
def test_add_file_to_zip(self):
tmp_dir1 = tempfile.mkdtemp()
zip_file = tmp_dir1 + "/foo.zip"
_, tmp_file = tempfile.mkstemp()
with open(tmp_file, "w") as tmpf:
tmpf.write("hello world")
with zipfile.ZipFile(zip_file, "w"):
pass # create empty archive
file_utils.add_file_to_zip(zip_file, tmp_file,
os.path.basename(tmp_file))
with zipfile.ZipFile(zip_file, "r") as zf:
with zf.open(os.path.basename(tmp_file), "r") as f:
self.assertEqual(f.read().decode("ascii"), "hello world")
if trans[ 'to' ] == 'html':
if trans[ 'tool' ] == 'tx':
#txResp = myTx( awsid, tmpFileDir, preConvertBucket, pusher, api_url, payload )
# def myTx( aswid, massagedFilesDir, preConvertBucket, authorUsername, api_url, data ):
# Zip up the massaged files
zipFilename = awsid + '.zip' # context.aws_request_id is a unique ID
# for this lambda call, so using it to not conflict with other requests
zipFile = os.path.join( tempfile.gettempdir(), zipFilename )
myLog( "info", "zipFile: " + zipFile )
mdFiles = glob(os.path.join( tmpFileDir, '*.md' ))
print('Zipping files from {0} to {1}...'.format( tmpFileDir, zipFile ), end=' ' )
fileCount = 0
for mdFile in mdFiles:
add_file_to_zip(zipFile, mdFile, os.path.basename( mdFile ))
fileCount += 1
print( 'finished zipping: ' + str( fileCount ) + " files." )
# 4) Upload zipped file to the S3 bucket (you may want to do some try/catch
# and give an error if fails back to Gogs)
print('Uploading {0} to {1} in {2}...'.format( zipFile, preConvertBucket, zipFilename ), end=' ')
s3Client = boto3.client('s3')
s3Client.upload_file( zipFile, preConvertBucket, zipFilename )
print( 'finished upload.' )
# Send job request to tx-manager
sourceUrl = 'https://s3-us-west-2.amazonaws.com/' + preConvertBucket + '/' + zipFilename # we use us-west-2 for our s3 buckets
txManagerJobUrl = api_url + '/tx/job'
gogsUserToken = payload[ 'secret' ]
def handle(event, context):
# Get vars and data
env_vars = retrieve(event, 'vars', 'payload')
api_url = retrieve(env_vars, 'api_url', 'Environment Vars')
pre_convert_bucket = retrieve(env_vars, 'pre_convert_bucket',
'Environment Vars')
cdn_bucket = retrieve(env_vars, 'cdn_bucket', 'Environment Vars')
gogs_url = retrieve(env_vars, 'gogs_url', 'Environment Vars')
gogs_user_token = retrieve(env_vars, 'gogs_user_token', 'Environment Vars')
repo_commit = retrieve(event, 'data', 'payload')
commit_id = repo_commit['after']
commit = None
for commit in repo_commit['commits']:
if commit['id'] == commit_id:
break
commit_url = commit['url']
commit_message = commit['message']
if gogs_url not in commit_url:
raise Exception(
'Repos can only belong to {0} to use this webhook client.'.format(
gogs_url))
repo_name = repo_commit['repository']['name']
repo_owner = repo_commit['repository']['owner']['username']
compare_url = repo_commit['compare_url']
if 'pusher' in repo_commit:
pusher = repo_commit['pusher']
else:
pusher = {'username': commit['author']['username']}
pusher_username = pusher['username']
# 1) Download and unzip the repo files
temp_dir = tempfile.mkdtemp(prefix='repo_')
download_repo(commit_url, temp_dir)
repo_dir = os.path.join(temp_dir, repo_name)
if not os.path.isdir(repo_dir):
repo_dir = temp_dir
# 2) Get the manifest file or make one if it doesn't exist based on meta.json, repo_name and file extensions
manifest_path = os.path.join(repo_dir, 'manifest.json')
if not os.path.isfile(manifest_path):
manifest_path = os.path.join(repo_dir, 'project.json')
if not os.path.isfile(manifest_path):
manifest_path = None
meta_path = os.path.join(repo_dir, 'meta.json')
meta = None
if os.path.isfile(meta_path):
meta = MetaData(meta_path)
manifest = Manifest(file_name=manifest_path,
repo_name=repo_name,
files_path=repo_dir,
meta=meta)
# determining the repo compiler:
generator = ''
if manifest.generator and manifest.generator[
'name'] and manifest.generator['name'].startswith('ts'):
generator = 'ts'
if not generator:
dirs = sorted(get_subdirs(repo_dir, True))
if 'content' in dirs:
repo_dir = os.path.join(repo_dir, 'content')
elif 'usfm' in dirs:
repo_dir = os.path.join(repo_dir, 'usfm')
manifest_path = os.path.join(repo_dir, 'manifest.json')
write_file(manifest_path, manifest.__dict__
) # Write it back out so it's using the latest manifest format
input_format = manifest.format
resource_type = manifest.resource['id']
if resource_type == 'ulb' or resource_type == 'udb':
resource_type = 'bible'
print(generator)
print(input_format)
print(manifest.__dict__)
try:
compiler_class = str_to_class(
'preprocessors.{0}{1}{2}Preprocessor'.format(
generator.capitalize(), resource_type.capitalize(),
input_format.capitalize()))
except AttributeError as e:
print('Got AE: {0}'.format(e.message))
compiler_class = preprocessors.Preprocessor
print(compiler_class)
# merge the source files with the template
output_dir = tempfile.mkdtemp(prefix='output_')
compiler = compiler_class(manifest, repo_dir, output_dir)
compiler.run()
# 3) Zip up the massaged files
zip_filename = context.aws_request_id + '.zip' # context.aws_request_id is a unique ID for this lambda call, so using it to not conflict with other requests
zip_filepath = os.path.join(tempfile.gettempdir(), zip_filename)
print('Zipping files from {0} to {1}...'.format(output_dir, zip_filepath))
add_contents_to_zip(zip_filepath, output_dir)
if os.path.isfile(manifest_path) and not os.path.isfile(
os.path.join(output_dir, 'manifest.json')):
add_file_to_zip(zip_filepath, manifest_path, 'manifest.json')
print('finished.')
# 4) Upload zipped file to the S3 bucket (you may want to do some try/catch and give an error if fails back to Gogs)
s3_handler = S3Handler(pre_convert_bucket)
file_key = "preconvert/" + zip_filename
print('Uploading {0} to {1}/{2}...'.format(zip_filepath,
pre_convert_bucket, file_key))
s3_handler.upload_file(zip_filepath, file_key)
print('finished.')
# Send job request to tx-manager
source_url = 'https://s3-us-west-2.amazonaws.com/{0}/{1}'.format(
pre_convert_bucket, file_key) # we use us-west-2 for our s3 buckets
tx_manager_job_url = api_url + '/tx/job'
identifier = "{0}/{1}/{2}".format(
repo_owner, repo_name, commit_id[:10]
) # The way to know which repo/commit goes to this job request
if input_format == 'markdown':
input_format = 'md'
payload = {
"identifier": identifier,
"user_token": gogs_user_token,
"resource_type": manifest.resource['id'],
"input_format": input_format,
"output_format": "html",
"source": source_url,
"callback": api_url + '/client/callback'
}
headers = {"content-type": "application/json"}
print('Making request to tx-Manager URL {0} with payload:'.format(
tx_manager_job_url))
print(payload)
print('...')
response = requests.post(tx_manager_job_url, json=payload, headers=headers)
print('finished.')
# for testing
print('tx-manager response:')
print(response)
if not response:
raise Exception('Bad request: unable to convert')
if 'errorMessage' in response:
raise Exception('Bad request: {0}'.format(response['errorMessage']))
json_data = json.loads(response.text)
if 'errorMessage' in json_data:
raise Exception('Bad request: {0}'.format(json_data['errorMessage']))
if 'job' not in json_data:
raise Exception(
'Bad request: tX Manager did not return any info about the job request.'
)
build_log_json = json_data['job']
build_log_json['repo_name'] = repo_name
build_log_json['repo_owner'] = repo_owner
build_log_json['commit_id'] = commit_id
build_log_json['committed_by'] = pusher_username
build_log_json['commit_url'] = commit_url
build_log_json['compare_url'] = compare_url
build_log_json['commit_message'] = commit_message
if 'errorMessage' in json_data:
build_log_json['status'] = 'failed'
build_log_json['message'] = json_data['errorMessage']
# Upload files to S3:
# S3 location vars
cdn_handler = S3Handler(cdn_bucket)
s3_commit_key = 'u/{0}'.format(identifier)
# Remove everything in the bucket with the s3_commit_key prefix so old files are removed, if any
for obj in cdn_handler.get_objects(prefix=s3_commit_key):
cdn_handler.delete_file(obj.key)
# Make a build_log.json file with this repo and commit data for later processing, upload to S3
build_log_file = os.path.join(tempfile.gettempdir(),
'build_log_request.json')
write_file(build_log_file, build_log_json)
cdn_handler.upload_file(build_log_file, s3_commit_key + '/build_log.json',
0)
# Upload the manifest.json file to the cdn_bucket if it exists
if os.path.isfile(manifest_path):
cdn_handler.upload_file(manifest_path,
s3_commit_key + '/manifest.json', 0)
# If there was an error, in order to trigger a 400 error in the API Gateway, we need to raise an
# exception with the returned 'errorMessage' because the API Gateway needs to see 'Bad Request:' in the string
if 'errorMessage' in json_data:
raise Exception('Bad Request: {0}'.format(json_data['errorMessage']))
return build_log_json
def handle(event, context):
try:
# Get vars and data
env_vars = retrieve(event, 'vars', 'payload')
api_url = retrieve(env_vars, 'api_url', 'Environment Vars')
pre_convert_bucket = retrieve(env_vars, 'pre_convert_bucket',
'Environment Vars')
cdn_bucket = retrieve(env_vars, 'cdn_bucket', 'Environment Vars')
gogs_url = retrieve(env_vars, 'gogs_url', 'Environment Vars')
gogs_user_token = retrieve(env_vars, 'gogs_user_token',
'Environment Vars')
repo_commit = retrieve(event, 'data', 'payload')
commit_id = repo_commit['after']
commit = None
for commit in repo_commit['commits']:
if commit['id'] == commit_id:
break
commit_id = commit_id[:10] # Only use the short form
commit_url = commit['url']
commit_message = commit['message']
if gogs_url not in commit_url:
raise Exception(
'Repos can only belong to {0} to use this webhook client.'.
format(gogs_url))
repo_name = repo_commit['repository']['name']
repo_owner = repo_commit['repository']['owner']['username']
compare_url = repo_commit['compare_url']
if 'pusher' in repo_commit:
pusher = repo_commit['pusher']
else:
pusher = {'username': commit['author']['username']}
pusher_username = pusher['username']
# 1) Download and unzip the repo files
temp_dir = tempfile.mkdtemp(prefix='repo_')
download_repo(commit_url, temp_dir)
repo_dir = os.path.join(temp_dir, repo_name)
if not os.path.isdir(repo_dir):
repo_dir = temp_dir
# 2) Get the manifest file or make one if it doesn't exist based on meta.json, repo_name and file extensions
manifest_path = os.path.join(repo_dir, 'manifest.json')
if not os.path.isfile(manifest_path):
manifest_path = os.path.join(repo_dir, 'project.json')
if not os.path.isfile(manifest_path):
manifest_path = None
meta_path = os.path.join(repo_dir, 'meta.json')
meta = None
if os.path.isfile(meta_path):
meta = MetaData(meta_path)
manifest = Manifest(file_name=manifest_path,
repo_name=repo_name,
files_path=repo_dir,
meta=meta)
# determining the repo compiler:
generator = ''
if manifest.generator and manifest.generator[
'name'] and manifest.generator['name'].startswith('ts'):
generator = 'ts'
if not generator:
dirs = sorted(get_subdirs(repo_dir, True))
if 'content' in dirs:
repo_dir = os.path.join(repo_dir, 'content')
elif 'usfm' in dirs:
repo_dir = os.path.join(repo_dir, 'usfm')
manifest_path = os.path.join(repo_dir, 'manifest.json')
write_file(
manifest_path, manifest.__dict__
) # Write it back out so it's using the latest manifest format
input_format = manifest.format
resource_type = manifest.resource['id']
if resource_type == 'ulb' or resource_type == 'udb':
resource_type = 'bible'
print(generator)
print(input_format)
print(manifest.__dict__)
try:
compiler_class = str_to_class(
'preprocessors.{0}{1}{2}Preprocessor'.format(
generator.capitalize(), resource_type.capitalize(),
input_format.capitalize()))
except AttributeError as e:
print('Got AE: {0}'.format(e.message))
compiler_class = preprocessors.Preprocessor
print(compiler_class)
# merge the source files with the template
output_dir = tempfile.mkdtemp(prefix='output_')
compiler = compiler_class(manifest, repo_dir, output_dir)
compiler.run()
# 3) Zip up the massaged files
zip_filename = context.aws_request_id + '.zip' # context.aws_request_id is a unique ID for this lambda call, so using it to not conflict with other requests
zip_filepath = os.path.join(tempfile.gettempdir(), zip_filename)
print('Zipping files from {0} to {1}...'.format(
output_dir, zip_filepath))
add_contents_to_zip(zip_filepath, output_dir)
if os.path.isfile(manifest_path) and not os.path.isfile(
os.path.join(output_dir, 'manifest.json')):
add_file_to_zip(zip_filepath, manifest_path, 'manifest.json')
print('finished.')
# 4) Upload zipped file to the S3 bucket (you may want to do some try/catch and give an error if fails back to Gogs)
s3_handler = S3Handler(pre_convert_bucket)
file_key = "preconvert/" + zip_filename
print('Uploading {0} to {1}/{2}...'.format(zip_filepath,
pre_convert_bucket,
file_key))
s3_handler.upload_file(zip_filepath, file_key)
print('finished.')
# Send job request to tx-manager
source_url = 'https://s3-us-west-2.amazonaws.com/{0}/{1}'.format(
pre_convert_bucket,
file_key) # we use us-west-2 for our s3 buckets
callback_url = api_url + '/client/callback'
tx_manager_job_url = api_url + '/tx/job'
identifier = "{0}/{1}/{2}".format(
repo_owner, repo_name, commit_id
) # The way to know which repo/commit goes to this job request
if input_format == 'markdown':
input_format = 'md'
payload = {
"identifier": identifier,
"user_token": gogs_user_token,
"resource_type": manifest.resource['id'],
"input_format": input_format,
"output_format": "html",
"source": source_url,
"callback": callback_url
}
headers = {"content-type": "application/json"}
print('Making request to tx-Manager URL {0} with payload:'.format(
tx_manager_job_url))
print(payload)
response = requests.post(tx_manager_job_url,
json=payload,
headers=headers)
print('finished.')
# for testing
print('tx-manager response:')
print(response)
print(response.status_code)
# Fake job in case tx-manager returns an error, can still build the build_log.json
job = {
'job_id': None,
'identifier': identifier,
'resource_type': manifest.resource['id'],
'input_format': input_format,
'output_format': 'html',
'source': source_url,
'callback': callback_url,
'message': 'Conversion started...',
'status': 'requested',
'success': None,
'created_at': datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
'log': [],
'warnings': [],
'errors': []
}
if response.status_code != requests.codes.ok:
job['status'] = 'failed'
job['success'] = False
job['message'] = 'Failed to convert!'
error = ''
if response.text:
try:
json_data = json.loads(response.text)
if 'errorMessage' in json_data:
error = json_data['errorMessage']
if error.startswith('Bad Request: '):
error = error[len('Bad Request: '):]
except Exception:
pass
job['errors'].append(error)
else:
json_data = json.loads(response.text)
if 'job' not in json_data:
job['status'] = 'failed'
job['success'] = False
job['message'] = 'Failed to convert'
job['errors'].append(
'tX Manager did not return any info about the job request.'
)
else:
job = json_data['job']
cdn_handler = S3Handler(cdn_bucket)
# Download the project.json file for this repo (create it if doesn't exist) and update it
project_json_key = 'u/{0}/{1}/project.json'.format(
repo_owner, repo_name)
project_json = cdn_handler.get_json(project_json_key)
project_json['user'] = repo_owner
project_json['repo'] = repo_name
project_json['repo_url'] = 'https://git.door43.org/{0}/{1}'.format(
repo_owner, repo_name)
commit = {
'id': commit_id,
'created_at': job['created_at'],
'status': job['status'],
'success': job['success'],
'started_at': None,
'ended_at': None
}
if 'commits' not in project_json:
project_json['commits'] = []
commits = []
for c in project_json['commits']:
if c['id'] != commit_id:
commits.append(c)
commits.append(commit)
project_json['commits'] = commits
project_file = os.path.join(tempfile.gettempdir(), 'project.json')
write_file(project_file, project_json)
cdn_handler.upload_file(project_file, project_json_key, 0)
# Compile data for build_log.json
build_log_json = job
build_log_json['repo_name'] = repo_name
build_log_json['repo_owner'] = repo_owner
build_log_json['commit_id'] = commit_id
build_log_json['committed_by'] = pusher_username
build_log_json['commit_url'] = commit_url
build_log_json['compare_url'] = compare_url
build_log_json['commit_message'] = commit_message
# Upload build_log.json and manifest.json to S3:
s3_commit_key = 'u/{0}'.format(identifier)
for obj in cdn_handler.get_objects(prefix=s3_commit_key):
cdn_handler.delete_file(obj.key)
build_log_file = os.path.join(tempfile.gettempdir(), 'build_log.json')
write_file(build_log_file, build_log_json)
cdn_handler.upload_file(build_log_file,
s3_commit_key + '/build_log.json', 0)
cdn_handler.upload_file(manifest_path,
s3_commit_key + '/manifest.json', 0)
if len(job['errors']) > 0:
raise Exception('; '.join(job['errors']))
else:
return build_log_json
except Exception as e:
raise Exception('Bad Request: {0}'.format(e))
def handle(event, context):
data = event['data']
commit_id = data['after']
commit = None
for commit in data['commits']:
if commit['id'] == commit_id:
break
commit_url = commit['url']
commit_message = commit['message']
if 'https://git.door43.org/' not in commit_url and 'http://test.door43.org:3000/' not in commit_url:
raise Exception('Currently only git.door43.org repositories are supported.')
pre_convert_bucket = event['pre_convert_bucket']
cdn_bucket = event['cdn_bucket']
gogs_user_token = event['gogs_user_token']
api_url = event['api_url']
repo_name = data['repository']['name']
repo_owner = data['repository']['owner']['username']
compare_url = data['compare_url']
if 'pusher' in data:
pusher = data['pusher']
else:
pusher = {'username': commit['author']['username']}
pusher_username = pusher['username']
# The following sections of code will:
# 1) download and unzip repo files
# 2) massage the repo files by creating a new directory and file structure
# 3) zip up the massages filed
# 4) upload massaged files to S3 in zip file
# 1) Download and unzip the repo files
repo_zip_url = commit_url.replace('commit', 'archive') + '.zip'
repo_zip_file = os.path.join(tempfile.gettempdir(), repo_zip_url.rpartition('/')[2])
repo_dir = tempfile.mkdtemp(prefix='repo_')
try:
print('Downloading {0}...'.format(repo_zip_url), end=' ')
if not os.path.isfile(repo_zip_file):
download_file(repo_zip_url, repo_zip_file)
finally:
print('finished.')
# Unzip the archive
try:
print('Unzipping {0}...'.format(repo_zip_file), end=' ')
unzip(repo_zip_file, repo_dir)
finally:
print('finished.')
# 2) Massage the content to just be a directory of MD files in alphabetical order as they should be compiled together in the converter
content_dir = os.path.join(repo_dir, repo_name, 'content')
md_files = glob(os.path.join(content_dir, '*.md'))
massaged_files_dir = tempfile.mktemp(prefix='files_')
make_dir(massaged_files_dir)
print('Massaging content from {0} to {1}...'.format(content_dir, massaged_files_dir), end=' ')
for md_file in md_files:
copyfile(md_file, os.path.join(massaged_files_dir, os.path.basename(md_file)))
# want front matter to be before 01.md and back matter to be after 50.md
copyfile(os.path.join(content_dir, '_front', 'front-matter.md'), os.path.join(massaged_files_dir, '00_front-matter.md'))
copyfile(os.path.join(content_dir, '_back', 'back-matter.md'), os.path.join(massaged_files_dir, '51_back-matter.md'))
print('finished.')
# 3) Zip up the massaged files
zip_filename = context.aws_request_id+'.zip' # context.aws_request_id is a unique ID for this lambda call, so using it to not conflict with other requests
zip_filepath = os.path.join(tempfile.gettempdir(), zip_filename)
md_files = glob(os.path.join(massaged_files_dir, '*.md'))
print('Zipping files from {0} to {1}...'.format(massaged_files_dir, zip_filepath), end=' ')
for md_file in md_files:
add_file_to_zip(zip_filepath, md_file, os.path.basename(md_file))
print('finished.')
# 4) Upload zipped file to the S3 bucket (you may want to do some try/catch and give an error if fails back to Gogs)
print('Uploading {0} to {1}...'.format(zip_filepath, pre_convert_bucket), end=' ')
s3_client = boto3.client('s3')
s3_client.upload_file(zip_filepath, pre_convert_bucket, zip_filename)
print('finished.')
# Send job request to tx-manager
source_url = 'https://s3-us-west-2.amazonaws.com/'+pre_convert_bucket+'/'+zip_filename # we use us-west-2 for our s3 buckets
tx_manager_job_url = api_url+'/tx/job'
payload = {
"identifier": "{0}:::{1}:::{2}".format(repo_owner, repo_name, commit_id),
"user_token": gogs_user_token,
"username": pusher_username,
"resource_type": "obs",
"input_format": "md",
"output_format": "html",
"source": source_url,
"callback": api_url+'/sampleclient/callback'
}
headers = {"content-type": "application/json"}
print('Making request to tx-Manager URL {0} with payload:'.format(tx_manager_job_url))
print(payload)
print('...', end=' ')
response = requests.post(tx_manager_job_url, json=payload, headers=headers)
print('finished.')
# for testing
print('tx-manager response:')
print(response)
json_data = json.loads(response.text)
print("json:")
print(json_data)
build_log_json = {
'job_id': json_data['job']['job_id'],
'repo_name': repo_name,
'repo_owner': repo_owner,
'commit_id': commit_id,
'committed_by': pusher_username,
'commit_url': commit_url,
'compare_url': compare_url,
'commit_message': commit_message,
'request_timestamp': datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
'eta_timestamp': json_data['job']['eta'],
'success': None,
'status': 'started',
'message': 'Conversion in progress...'
}
if 'errorMessage' in json_data:
build_log_json['status'] = 'failed'
build_log_json['message'] = json_data['errorMessage']
# Make a build_log.json file with this repo and commit data for later processing, upload to S3
s3_project_key = 'u/{0}/{1}/{2}'.format(repo_owner, repo_name, commit_id[:10])
s3_resource = boto3.resource('s3')
bucket = s3_resource.Bucket(cdn_bucket)
for obj in bucket.objects.filter(Prefix=s3_project_key):
s3_resource.Object(bucket.name, obj.key).delete()
build_log_file = os.path.join(tempfile.gettempdir(), 'build_log_request.json')
write_file(build_log_file, build_log_json)
bucket.upload_file(build_log_file, s3_project_key+'/build_log.json', ExtraArgs={'ContentType': 'application/json'})
print('Uploaded the following content from {0} to {1}/build_log.json'.format(build_log_file, s3_project_key))
print(build_log_json)
# If there was an error, in order to trigger a 400 error in the API Gateway, we need to raise an
# exception with the returned 'errorMessage' because the API Gateway needs to see 'Bad Request:' in the string
if 'errorMessage' in json_data:
raise Exception(json_data['errorMessage'])
return build_log_json