def export_user_projects(user_id):
user = User.objects.get(pk=user_id)
projects = Project.objects.filter(owner=user)
with tempfile.NamedTemporaryFile(suffix='.zip', delete=False) as temp:
filename = temp.name
with zipfile.ZipFile(filename, 'w',
compression=zipfile.ZIP_DEFLATED) as z:
for project in projects:
add_project_to_archive(z,
project,
prefix='cloudpebble-export/',
suffix='-%d' % project.id)
send_td_event('cloudpebble_export_all_projects', user=user)
# Generate a URL
u = uuid.uuid4().hex
if not settings.AWS_ENABLED:
outfile = '%s%s/%s.zip' % (settings.EXPORT_DIRECTORY, u,
'cloudpebble-export')
os.makedirs(os.path.dirname(outfile), 0755)
shutil.copy(filename, outfile)
os.chmod(outfile, 0644)
return '%s%s/%s.zip' % (settings.EXPORT_ROOT, u,
'cloudpebble-export')
else:
outfile = '%s/%s.zip' % (u, 'cloudpebble-export')
s3.upload_file('export',
outfile,
filename,
public=True,
content_type='application/zip')
return '%s%s' % (settings.EXPORT_ROOT, outfile)
def save_pbw(self, pbw_path):
s3.upload_file('builds',
self.pbw,
pbw_path,
public=True,
download_filename='%s.pbw' %
self.project.app_short_name.replace('/', '-'))
def create_archive(project_id):
project = Project.objects.get(pk=project_id)
prefix = re.sub(r'[^\w]+', '_', project.name).strip('_').lower()
with tempfile.NamedTemporaryFile(suffix='.zip', delete=False) as temp:
filename = temp.name
with zipfile.ZipFile(filename, 'w',
compression=zipfile.ZIP_DEFLATED) as z:
add_project_to_archive(z, project)
# Generate a URL
u = uuid.uuid4().hex
send_td_event('cloudpebble_export_project', project=project)
if not settings.AWS_ENABLED:
outfile = '%s%s/%s.zip' % (settings.EXPORT_DIRECTORY, u, prefix)
os.makedirs(os.path.dirname(outfile), 0755)
shutil.copy(filename, outfile)
os.chmod(outfile, 0644)
return '%s%s/%s.zip' % (settings.EXPORT_ROOT, u, prefix)
else:
outfile = '%s/%s.zip' % (u, prefix)
s3.upload_file('export',
outfile,
filename,
public=True,
content_type='application/zip')
return '%s%s' % (settings.EXPORT_ROOT, outfile)
def render_snapshot(source_location,
organization_id,
project_id,
build_id,
title,
width,
browser,
selector,
hide_selectors,
save_snapshot=True):
key_path = '{}/{}/{}'.format(organization_id, project_id, build_id)
if not browser == render.browser and render.browser is not None:
render.close_browser()
if not render.is_open:
render.open_browser(browser)
try:
image = render.render(source_location, width, selector, hide_selectors)
except:
render.close_browser()
raise
image_blob = io.BytesIO(image)
if save_snapshot:
file_name = '{}.html'.format(uuid.uuid4().hex)
image_key = '{}/screenshots/{}/{}/{}.png'.format(
key_path, browser, width, file_name)
upload_file(image_blob, image_key)
image_location = '{}/{}/{}'.format(S3_ENDPOINT, SCREENSHOT_BUCKET,
image_key)
else:
image_location = None
return io.BytesIO(image), image_location
def save_package(self, package_path):
filename = '%s.tar.gz' % self.project.app_short_name.replace('/', '-')
s3.upload_file('builds',
self.package,
package_path,
public=True,
download_filename=filename,
content_type='application/gzip')
def diff_snapshot(snapshot,
organization_id,
project_id,
build_id,
browser,
title,
width,
compare_snapshot,
flake_sha_list,
save_snapshot=False):
snapshot_key = urlparse(compare_snapshot).path.split('/', 2)[2]
old_snapshot = get_file(snapshot_key)
diff_snapshot, difference, diff_pixels = compare(old_snapshot, snapshot)
snapshot_location = compare_snapshot
key_path = '{}/{}/{}'.format(organization_id, project_id, build_id)
diff_matched = None
flake_matched = None
diff_hash = ''
diff_location = None
if difference > 0.1:
diff_key_path = "{}/diff/{}".format(
key_path,
width,
)
key = "{}/{}.png".format(
diff_key_path,
uuid.uuid4().hex,
)
snapshot_stream = io.BytesIO(diff_snapshot)
diff_hash = get_sha1(diff_snapshot)
flake_sha_set = set(flake_sha_list)
flake_matched = diff_hash in flake_sha_set
# flake_key_matched = match_snapshot_flake(diff_snapshot, flake_key_path)
# if flake_key_matched:
# flake_matched = '{}/{}/{}'.format(S3_ENDPOINT,
# SCREENSHOT_BUCKET, flake_key_matched)
if not flake_matched:
s3image = upload_file(snapshot_stream, key)
diff_location = '{}/{}/{}'.format(S3_ENDPOINT, SCREENSHOT_BUCKET,
key)
if save_snapshot:
file_name = '{}.html'.format(uuid.uuid4().hex)
image_key = '{}/screenshots/{}/{}/{}.png'.format(
key_path, browser, width, file_name)
snapshot.seek(0)
upload_file(snapshot, image_key)
snapshot_location = '{}/{}/{}'.format(S3_ENDPOINT,
SCREENSHOT_BUCKET, image_key)
else:
snapshot.close()
return diff_location, difference, snapshot_location, diff_hash, flake_matched
def save_pbw(self, pbw_path):
if not settings.AWS_ENABLED:
shutil.move(pbw_path, self.pbw)
else:
s3.upload_file('builds',
self.pbw,
pbw_path,
public=True,
download_filename='%s.pbw' %
self.project.app_short_name.replace('/', '-'))
def save_pbw(self, pbw_path):
if not settings.AWS_ENABLED:
shutil.move(pbw_path, self.pbw)
else:
s3.upload_file(
"builds",
self.pbw,
pbw_path,
public=True,
download_filename="%s.pbw" % self.project.app_short_name.replace("/", "-"),
)
def save_package(self, package_path):
if not settings.AWS_ENABLED:
shutil.move(package_path, self.package)
else:
filename = '%s.tar.gz' % self.project.app_short_name.replace(
'/', '-')
s3.upload_file('builds',
self.package,
package_path,
public=True,
download_filename=filename,
content_type='application/gzip')
def render_snapshot(source_location,
organization_id,
project_id,
build_id,
title,
width,
browser,
selector,
hide_selectors,
save_snapshot=True):
key_path = '{}/{}/{}'.format(organization_id, project_id, build_id)
if PRIVATE_ASSETS:
snapshot_source_key = urlparse(source_location).path.split('/', 2)[2]
url = "{}/{}/{}?token={}".format(BASSET_URL, "snapshot_source",
snapshot_source_key, TOKEN)
else:
url = source_location
if not browser == render.browser and render.browser is not None:
render.close_browser()
if not render.is_open:
render.open_browser(browser)
try:
image = render.render(url, width, selector, hide_selectors)
except:
render.close_browser()
raise
print("Rendered snapshot: {}".format(title))
print("[org ID: {}] [project ID: {}] [build ID: {}]".format(
organization_id, project_id, build_id))
print("[browser: {}] [width: {}]".format(browser, width))
image_blob = io.BytesIO(image)
if save_snapshot:
file_name = '{}.html'.format(uuid.uuid4().hex)
image_key = '{}/screenshots/{}/{}/{}.png'.format(
key_path, browser, width, file_name)
upload_file(image_blob, image_key)
image_location = '{}/{}/{}'.format(S3_ENDPOINT, SCREENSHOT_BUCKET,
image_key)
else:
image_location = None
return io.BytesIO(image), image_location
def create_archive(project_id):
project = Project.objects.get(pk=project_id)
prefix = re.sub(r"[^\w]+", "_", project.name).strip("_").lower()
with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as temp:
filename = temp.name
with zipfile.ZipFile(filename, "w", compression=zipfile.ZIP_DEFLATED) as z:
add_project_to_archive(z, project)
# Generate a URL
u = uuid.uuid4().hex
send_keen_event("cloudpebble", "cloudpebble_export_project", project=project)
if not settings.AWS_ENABLED:
outfile = "%s%s/%s.zip" % (settings.EXPORT_DIRECTORY, u, prefix)
os.makedirs(os.path.dirname(outfile), 0755)
shutil.copy(filename, outfile)
os.chmod(outfile, 0644)
return "%s%s/%s.zip" % (settings.EXPORT_ROOT, u, prefix)
else:
outfile = "%s/%s.zip" % (u, prefix)
s3.upload_file("export", outfile, filename, public=True, content_type="application/zip")
return "%s%s" % (settings.EXPORT_ROOT, outfile)
def export_user_projects(user_id):
user = User.objects.get(pk=user_id)
projects = Project.objects.filter(owner=user)
with tempfile.NamedTemporaryFile(suffix='.zip', delete=False) as temp:
filename = temp.name
with zipfile.ZipFile(filename, 'w', compression=zipfile.ZIP_DEFLATED) as z:
for project in projects:
add_project_to_archive(z, project, prefix='cloudpebble-export/', suffix='-%d' % project.id)
send_td_event('cloudpebble_export_all_projects', user=user)
# Generate a URL
u = uuid.uuid4().hex
if not settings.AWS_ENABLED:
outfile = '%s%s/%s.zip' % (settings.EXPORT_DIRECTORY, u, 'cloudpebble-export')
os.makedirs(os.path.dirname(outfile), 0755)
shutil.copy(filename, outfile)
os.chmod(outfile, 0644)
return '%s%s/%s.zip' % (settings.EXPORT_ROOT, u, 'cloudpebble-export')
else:
outfile = '%s/%s.zip' % (u, 'cloudpebble-export')
s3.upload_file('export', outfile, filename, public=True, content_type='application/zip')
return '%s%s' % (settings.EXPORT_ROOT, outfile)
def save_pbw(self, pbw_path):
if not settings.AWS_ENABLED:
shutil.move(pbw_path, self.pbw)
else:
s3.upload_file('builds', self.pbw, pbw_path, public=True, download_filename='%s.pbw' % self.project.app_short_name.replace('/','-'))
def main(pdf_files):
"""The entry-point main function."""
for pdf_file in pdf_files:
pdf_directory_registry = initialize_pdf_directory(
os.path.splitext(pdf_file[5:])[0])
copy_file_path = f"{pdf_directory_registry['root']}/index.pdf"
copy_file(pdf_file, copy_file_path)
manifest_file_path = f"{pdf_directory_registry['root']}/manifest.txt"
with open(manifest_file_path, 'w', encoding='utf-8') as manifest_file:
manifest_file.write('ARREST LOG MANIFEST\n')
pdf_file_pages = split_into_pages(pdf_directory_registry['pages'],
pdf_file)
image_files = []
for page in pdf_file_pages:
image_files.append(
convert_pdf_to_png(pdf_directory_registry['images'], page))
for i, img_file in enumerate(image_files):
# Crop the headers and footers out of each page
width, height = retrieve_image_dimensions(img_file)
if i == 0:
left, top, right, bottom = 0, 270, width, height - 100
else:
left, top, right, bottom = 0, 160, width, height - 90
crop_image(img_file, img_file, (left, top, right, bottom))
image_files[i] = cv2.imread(img_file)
concat_image = concat_images(pdf_directory_registry['images'],
image_files)
record_starting_points = retrieve_record_starting_points(concat_image)
print(f'# of records detected: {len(record_starting_points)}')
with open(manifest_file_path, 'a', encoding='utf-8') as manifest_file:
manifest_file.write(
f'# of records: {len(record_starting_points)}\n')
width, height = retrieve_image_dimensions(concat_image)
records = []
for i, starting_point in enumerate(record_starting_points):
left, top, right, bottom = 0, starting_point - 5, width, height
if i + 1 != len(record_starting_points):
bottom = record_starting_points[i + 1] - 5
records_subdir = pdf_directory_registry['records']
cropped_record_filename = f'{records_subdir}/{str(i + 1)}.png'
records.append(
crop_image(concat_image, cropped_record_filename,
(left, top, right, bottom)))
total_offenses_count = 0
for i, record_image_path in enumerate(records):
print(record_image_path)
record_data = {}
# START - ITERATE OVER STATIC CATEGORIES - START
static_categories = get_static_record_categories()
for category in static_categories:
top, bottom, left, right = get_dimensions_from_static_category(
category)
cropped_category_filename = \
f'{constants.PDF_TMP_DIRECTORY}/record_{str(i + 1)}_{category}.png'
crop_image(record_image_path, cropped_category_filename,
(left, top, right, bottom))
category_text = read_text(cropped_category_filename)
record_data[category] = clean_text(category_text)
# END - ITERATE OVER STATIC CATEGORIES - END
# START - ITERATE OVER DYNAMIC CATEGORIES - START
offenses_starting_points = retrieve_offense_starting_points(
record_image_path)
total_offenses_count += len(offenses_starting_points)
print(f'# of offenses detected: {len(offenses_starting_points)}')
offenses = []
for j, offense_starting_point in enumerate(
offenses_starting_points):
left, top, right, bottom = 0, offense_starting_point - 5, width, height
if j + 1 != len(offenses_starting_points):
bottom = offenses_starting_points[j + 1] - 5
cropped_offense_filename = \
f'{constants.PDF_TMP_DIRECTORY}/record_{str(i + 1)}_offense_{str(j + 1)}.png'
offenses.append(
crop_image(record_image_path, cropped_offense_filename,
(left, top, right, bottom)))
dynamic_categories = get_dynamic_record_categories()
for category in dynamic_categories:
record_data[category] = []
for offense_file_path in offenses:
for category in dynamic_categories:
top, bottom, left, right = get_dimensions_from_dynamic_category(
category)
cropped_category_filename = \
f'{constants.PDF_TMP_DIRECTORY}/record_{str(i + 1)}_{category}.png'
crop_image(offense_file_path, cropped_category_filename,
(left, top, right, bottom))
category_text = read_text(cropped_category_filename)
record_data[category].append(clean_text(category_text))
# END - ITERATE OVER DYNAMIC CATEGORIES - END
# Convert "ethnicities" string to an array of strings
# "White, Tongan" -> ["White", "Tongan"]
record_data['ethnicities'] = record_data['ethnicities'].split(',')
record_data['ethnicities'] = correct_ethnicities(
record_data['ethnicities'])
# Geolocate the locations
locations = []
for location in record_data['location_of_arrest']:
geolocated_location = geolocate_location(location)
locations.append(geolocated_location)
record_data['locations'] = locations
img_file_id = str(uuid.uuid4())
img_filename = f'{img_file_id}.png'
with open(record_image_path, 'rb') as file:
upload_file(constants.RECORDS_BUCKET_NAME, img_filename, file,
'image/png')
record_data['id'] = str(uuid.uuid4())
record_data['imageId'] = img_filename
print(record_data)
insert_item(constants.RECORDS_TABLE_NAME, {
'date': pdf_file.split('/')[1][0:-4],
**record_data
})
with open(manifest_file_path, 'a', encoding='utf-8') as manifest_file:
manifest_file.write(f'# of offenses: {total_offenses_count}\n')
output_zip_file = f"{constants.PDF_TMP_DIRECTORY}/{pdf_file.split('/')[1][0:-4]}"
zip_directory(output_zip_file, pdf_directory_registry['root'])
with open(output_zip_file + '.zip', 'rb') as file:
upload_file(constants.ARTIFACTS_BUCKET_NAME,
pdf_file.split('/')[1][0:-4] + '.zip', file,
'application/zip')
def save_package(self, package_path):
if not settings.AWS_ENABLED:
shutil.move(package_path, self.package)
else:
filename = '%s.tar.gz' % self.project.app_short_name.replace('/', '-')
s3.upload_file('builds', self.package, package_path, public=True, download_filename=filename, content_type='application/gzip')
