def export_user_projects(user_id): user = User.objects.get(pk=user_id) projects = Project.objects.filter(owner=user) with tempfile.NamedTemporaryFile(suffix='.zip', delete=False) as temp: filename = temp.name with zipfile.ZipFile(filename, 'w', compression=zipfile.ZIP_DEFLATED) as z: for project in projects: add_project_to_archive(z, project, prefix='cloudpebble-export/', suffix='-%d' % project.id) send_td_event('cloudpebble_export_all_projects', user=user) # Generate a URL u = uuid.uuid4().hex if not settings.AWS_ENABLED: outfile = '%s%s/%s.zip' % (settings.EXPORT_DIRECTORY, u, 'cloudpebble-export') os.makedirs(os.path.dirname(outfile), 0755) shutil.copy(filename, outfile) os.chmod(outfile, 0644) return '%s%s/%s.zip' % (settings.EXPORT_ROOT, u, 'cloudpebble-export') else: outfile = '%s/%s.zip' % (u, 'cloudpebble-export') s3.upload_file('export', outfile, filename, public=True, content_type='application/zip') return '%s%s' % (settings.EXPORT_ROOT, outfile)
def save_pbw(self, pbw_path): s3.upload_file('builds', self.pbw, pbw_path, public=True, download_filename='%s.pbw' % self.project.app_short_name.replace('/', '-'))
def create_archive(project_id): project = Project.objects.get(pk=project_id) prefix = re.sub(r'[^\w]+', '_', project.name).strip('_').lower() with tempfile.NamedTemporaryFile(suffix='.zip', delete=False) as temp: filename = temp.name with zipfile.ZipFile(filename, 'w', compression=zipfile.ZIP_DEFLATED) as z: add_project_to_archive(z, project) # Generate a URL u = uuid.uuid4().hex send_td_event('cloudpebble_export_project', project=project) if not settings.AWS_ENABLED: outfile = '%s%s/%s.zip' % (settings.EXPORT_DIRECTORY, u, prefix) os.makedirs(os.path.dirname(outfile), 0755) shutil.copy(filename, outfile) os.chmod(outfile, 0644) return '%s%s/%s.zip' % (settings.EXPORT_ROOT, u, prefix) else: outfile = '%s/%s.zip' % (u, prefix) s3.upload_file('export', outfile, filename, public=True, content_type='application/zip') return '%s%s' % (settings.EXPORT_ROOT, outfile)
def render_snapshot(source_location, organization_id, project_id, build_id, title, width, browser, selector, hide_selectors, save_snapshot=True): key_path = '{}/{}/{}'.format(organization_id, project_id, build_id) if not browser == render.browser and render.browser is not None: render.close_browser() if not render.is_open: render.open_browser(browser) try: image = render.render(source_location, width, selector, hide_selectors) except: render.close_browser() raise image_blob = io.BytesIO(image) if save_snapshot: file_name = '{}.html'.format(uuid.uuid4().hex) image_key = '{}/screenshots/{}/{}/{}.png'.format( key_path, browser, width, file_name) upload_file(image_blob, image_key) image_location = '{}/{}/{}'.format(S3_ENDPOINT, SCREENSHOT_BUCKET, image_key) else: image_location = None return io.BytesIO(image), image_location
def save_package(self, package_path): filename = '%s.tar.gz' % self.project.app_short_name.replace('/', '-') s3.upload_file('builds', self.package, package_path, public=True, download_filename=filename, content_type='application/gzip')
def diff_snapshot(snapshot, organization_id, project_id, build_id, browser, title, width, compare_snapshot, flake_sha_list, save_snapshot=False): snapshot_key = urlparse(compare_snapshot).path.split('/', 2)[2] old_snapshot = get_file(snapshot_key) diff_snapshot, difference, diff_pixels = compare(old_snapshot, snapshot) snapshot_location = compare_snapshot key_path = '{}/{}/{}'.format(organization_id, project_id, build_id) diff_matched = None flake_matched = None diff_hash = '' diff_location = None if difference > 0.1: diff_key_path = "{}/diff/{}".format( key_path, width, ) key = "{}/{}.png".format( diff_key_path, uuid.uuid4().hex, ) snapshot_stream = io.BytesIO(diff_snapshot) diff_hash = get_sha1(diff_snapshot) flake_sha_set = set(flake_sha_list) flake_matched = diff_hash in flake_sha_set # flake_key_matched = match_snapshot_flake(diff_snapshot, flake_key_path) # if flake_key_matched: # flake_matched = '{}/{}/{}'.format(S3_ENDPOINT, # SCREENSHOT_BUCKET, flake_key_matched) if not flake_matched: s3image = upload_file(snapshot_stream, key) diff_location = '{}/{}/{}'.format(S3_ENDPOINT, SCREENSHOT_BUCKET, key) if save_snapshot: file_name = '{}.html'.format(uuid.uuid4().hex) image_key = '{}/screenshots/{}/{}/{}.png'.format( key_path, browser, width, file_name) snapshot.seek(0) upload_file(snapshot, image_key) snapshot_location = '{}/{}/{}'.format(S3_ENDPOINT, SCREENSHOT_BUCKET, image_key) else: snapshot.close() return diff_location, difference, snapshot_location, diff_hash, flake_matched
def save_pbw(self, pbw_path): if not settings.AWS_ENABLED: shutil.move(pbw_path, self.pbw) else: s3.upload_file('builds', self.pbw, pbw_path, public=True, download_filename='%s.pbw' % self.project.app_short_name.replace('/', '-'))
def save_pbw(self, pbw_path): if not settings.AWS_ENABLED: shutil.move(pbw_path, self.pbw) else: s3.upload_file( "builds", self.pbw, pbw_path, public=True, download_filename="%s.pbw" % self.project.app_short_name.replace("/", "-"), )
def save_package(self, package_path): if not settings.AWS_ENABLED: shutil.move(package_path, self.package) else: filename = '%s.tar.gz' % self.project.app_short_name.replace( '/', '-') s3.upload_file('builds', self.package, package_path, public=True, download_filename=filename, content_type='application/gzip')
def render_snapshot(source_location, organization_id, project_id, build_id, title, width, browser, selector, hide_selectors, save_snapshot=True): key_path = '{}/{}/{}'.format(organization_id, project_id, build_id) if PRIVATE_ASSETS: snapshot_source_key = urlparse(source_location).path.split('/', 2)[2] url = "{}/{}/{}?token={}".format(BASSET_URL, "snapshot_source", snapshot_source_key, TOKEN) else: url = source_location if not browser == render.browser and render.browser is not None: render.close_browser() if not render.is_open: render.open_browser(browser) try: image = render.render(url, width, selector, hide_selectors) except: render.close_browser() raise print("Rendered snapshot: {}".format(title)) print("[org ID: {}] [project ID: {}] [build ID: {}]".format( organization_id, project_id, build_id)) print("[browser: {}] [width: {}]".format(browser, width)) image_blob = io.BytesIO(image) if save_snapshot: file_name = '{}.html'.format(uuid.uuid4().hex) image_key = '{}/screenshots/{}/{}/{}.png'.format( key_path, browser, width, file_name) upload_file(image_blob, image_key) image_location = '{}/{}/{}'.format(S3_ENDPOINT, SCREENSHOT_BUCKET, image_key) else: image_location = None return io.BytesIO(image), image_location
def create_archive(project_id): project = Project.objects.get(pk=project_id) prefix = re.sub(r"[^\w]+", "_", project.name).strip("_").lower() with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as temp: filename = temp.name with zipfile.ZipFile(filename, "w", compression=zipfile.ZIP_DEFLATED) as z: add_project_to_archive(z, project) # Generate a URL u = uuid.uuid4().hex send_keen_event("cloudpebble", "cloudpebble_export_project", project=project) if not settings.AWS_ENABLED: outfile = "%s%s/%s.zip" % (settings.EXPORT_DIRECTORY, u, prefix) os.makedirs(os.path.dirname(outfile), 0755) shutil.copy(filename, outfile) os.chmod(outfile, 0644) return "%s%s/%s.zip" % (settings.EXPORT_ROOT, u, prefix) else: outfile = "%s/%s.zip" % (u, prefix) s3.upload_file("export", outfile, filename, public=True, content_type="application/zip") return "%s%s" % (settings.EXPORT_ROOT, outfile)
def save_pbw(self, pbw_path): if not settings.AWS_ENABLED: shutil.move(pbw_path, self.pbw) else: s3.upload_file('builds', self.pbw, pbw_path, public=True, download_filename='%s.pbw' % self.project.app_short_name.replace('/','-'))
def main(pdf_files): """The entry-point main function.""" for pdf_file in pdf_files: pdf_directory_registry = initialize_pdf_directory( os.path.splitext(pdf_file[5:])[0]) copy_file_path = f"{pdf_directory_registry['root']}/index.pdf" copy_file(pdf_file, copy_file_path) manifest_file_path = f"{pdf_directory_registry['root']}/manifest.txt" with open(manifest_file_path, 'w', encoding='utf-8') as manifest_file: manifest_file.write('ARREST LOG MANIFEST\n') pdf_file_pages = split_into_pages(pdf_directory_registry['pages'], pdf_file) image_files = [] for page in pdf_file_pages: image_files.append( convert_pdf_to_png(pdf_directory_registry['images'], page)) for i, img_file in enumerate(image_files): # Crop the headers and footers out of each page width, height = retrieve_image_dimensions(img_file) if i == 0: left, top, right, bottom = 0, 270, width, height - 100 else: left, top, right, bottom = 0, 160, width, height - 90 crop_image(img_file, img_file, (left, top, right, bottom)) image_files[i] = cv2.imread(img_file) concat_image = concat_images(pdf_directory_registry['images'], image_files) record_starting_points = retrieve_record_starting_points(concat_image) print(f'# of records detected: {len(record_starting_points)}') with open(manifest_file_path, 'a', encoding='utf-8') as manifest_file: manifest_file.write( f'# of records: {len(record_starting_points)}\n') width, height = retrieve_image_dimensions(concat_image) records = [] for i, starting_point in enumerate(record_starting_points): left, top, right, bottom = 0, starting_point - 5, width, height if i + 1 != len(record_starting_points): bottom = record_starting_points[i + 1] - 5 records_subdir = pdf_directory_registry['records'] cropped_record_filename = f'{records_subdir}/{str(i + 1)}.png' records.append( crop_image(concat_image, cropped_record_filename, (left, top, right, bottom))) total_offenses_count = 0 for i, record_image_path in enumerate(records): print(record_image_path) record_data = {} # START - ITERATE OVER STATIC CATEGORIES - START static_categories = get_static_record_categories() for category in static_categories: top, bottom, left, right = get_dimensions_from_static_category( category) cropped_category_filename = \ f'{constants.PDF_TMP_DIRECTORY}/record_{str(i + 1)}_{category}.png' crop_image(record_image_path, cropped_category_filename, (left, top, right, bottom)) category_text = read_text(cropped_category_filename) record_data[category] = clean_text(category_text) # END - ITERATE OVER STATIC CATEGORIES - END # START - ITERATE OVER DYNAMIC CATEGORIES - START offenses_starting_points = retrieve_offense_starting_points( record_image_path) total_offenses_count += len(offenses_starting_points) print(f'# of offenses detected: {len(offenses_starting_points)}') offenses = [] for j, offense_starting_point in enumerate( offenses_starting_points): left, top, right, bottom = 0, offense_starting_point - 5, width, height if j + 1 != len(offenses_starting_points): bottom = offenses_starting_points[j + 1] - 5 cropped_offense_filename = \ f'{constants.PDF_TMP_DIRECTORY}/record_{str(i + 1)}_offense_{str(j + 1)}.png' offenses.append( crop_image(record_image_path, cropped_offense_filename, (left, top, right, bottom))) dynamic_categories = get_dynamic_record_categories() for category in dynamic_categories: record_data[category] = [] for offense_file_path in offenses: for category in dynamic_categories: top, bottom, left, right = get_dimensions_from_dynamic_category( category) cropped_category_filename = \ f'{constants.PDF_TMP_DIRECTORY}/record_{str(i + 1)}_{category}.png' crop_image(offense_file_path, cropped_category_filename, (left, top, right, bottom)) category_text = read_text(cropped_category_filename) record_data[category].append(clean_text(category_text)) # END - ITERATE OVER DYNAMIC CATEGORIES - END # Convert "ethnicities" string to an array of strings # "White, Tongan" -> ["White", "Tongan"] record_data['ethnicities'] = record_data['ethnicities'].split(',') record_data['ethnicities'] = correct_ethnicities( record_data['ethnicities']) # Geolocate the locations locations = [] for location in record_data['location_of_arrest']: geolocated_location = geolocate_location(location) locations.append(geolocated_location) record_data['locations'] = locations img_file_id = str(uuid.uuid4()) img_filename = f'{img_file_id}.png' with open(record_image_path, 'rb') as file: upload_file(constants.RECORDS_BUCKET_NAME, img_filename, file, 'image/png') record_data['id'] = str(uuid.uuid4()) record_data['imageId'] = img_filename print(record_data) insert_item(constants.RECORDS_TABLE_NAME, { 'date': pdf_file.split('/')[1][0:-4], **record_data }) with open(manifest_file_path, 'a', encoding='utf-8') as manifest_file: manifest_file.write(f'# of offenses: {total_offenses_count}\n') output_zip_file = f"{constants.PDF_TMP_DIRECTORY}/{pdf_file.split('/')[1][0:-4]}" zip_directory(output_zip_file, pdf_directory_registry['root']) with open(output_zip_file + '.zip', 'rb') as file: upload_file(constants.ARTIFACTS_BUCKET_NAME, pdf_file.split('/')[1][0:-4] + '.zip', file, 'application/zip')
def save_package(self, package_path): if not settings.AWS_ENABLED: shutil.move(package_path, self.package) else: filename = '%s.tar.gz' % self.project.app_short_name.replace('/', '-') s3.upload_file('builds', self.package, package_path, public=True, download_filename=filename, content_type='application/gzip')