def settings(self) -> Objectify: """Project settings. Aggregate settings information about a project. :return: Dictionary with all settings for a project. """ # TODO: These settings are in a transitional state while # we move other configuration-related fields here. # To preserve backwards compability, we simply proxy those # fields - but their use should be deprecated as possible. # (NB. Even with Objectify, there is no provision # for write-back any of the "dates" subfields yet) return Objectify({ 'tech_requirements': self.tech_requirements, 'delivery_config': self.delivery, 'dates': { 'cancellation_window': self.cancellation_window, 'availability_window': self.availability_window, 'approval_window': self.approval_window, }, 'permissions': { 'add_order': self.add_order_roles }, 'order_type': self.order_type, 'project_type': self.project_type })
def create_collections(self, order_payload: dict) -> dict: """Create all collections in Alexandria if the do not exists. :param self: reference to the task class instance :param order_payload: payload of order from leica :return: order collection payload from the library """ factory = getUtility(IRemoteRestEndpoint) library_api = factory(config.ALEXANDRIA_BASE, 'collections', 'Collections') order = Objectify(order_payload) collections = [ (order.customer, 'customer'), (order.project, 'project'), (order, 'order'), ] parent_id = config.ALEXANDRIA_LEICA_ROOT for item, type_ in collections: result = library_api.get(item.id) if not result: payload = { 'slug': item.slug, 'id': item.id, 'title': item.title, 'description': item.description, 'content_type': f'application/collection.leica-{type_}', 'parent_id': parent_id, 'tags': [type_] } result = library_api.post(payload) # this will be the new parent parent_id = result.get('id') if order.requirement_items: for i, item in enumerate(order.requirement_items): result = library_api.get(item.id) if not result: category_name = item._get('name', f'ItemName-{i}') payload = { 'slug': slugify(category_name), 'id': item.id, 'title': item.category, 'description': item._get('description', ''), 'content_type': 'application/collection.leica-order.requirement', 'parent_id': order.id, 'tags': item.tags, 'properties': { 'gdrive': { 'folder_id': item.folder_id, 'parent_folder_id': item.parent_folder_id, 'created_by': '9df18a79-44dc-4c2f-86aa-09bf7706ae86' } } } library_api.post(payload) return library_api.get(order.id)
def main(): """Execute import script.""" for item in PROJECTS_TO_IMPORT: if not item.get('imported', False): session = db_configure(Session) importer = OrderImporter(item.get('file_name'), Objectify(item.get('project')), session) with transaction.manager: importer()
def process_message(self, message: SQSMessage) -> bool: """Process a message retrieved from the input_queue. :param message: A message from the queue :returns: Status from the process """ body = message.body data = Objectify(body.get('data', {})) event = body.get('event_name', '') message_id = body.get('id', '') data.sqs_message_id = message_id dispatch = Objectify(self.dispatch_map.get(event, {})) if not dispatch: logger.info('Unknown event type - message {0} ignored'.format( body['id'])) return True logger.info('Processing event {event}'.format(event=event)) try: status, payload = dispatch.action(data) except Exception as error: msg = 'Unknown exception raised on \'{0}\' assignment {1}. \n' \ 'Error: {2} \n Payload: {3}' logger.error(msg.format(dispatch.name, data.id, error, data._dct)) raise # Let newrelic deal with it. response = ResponseWrapper(data, payload) notification_action = dispatch.notification_actions[status] event = notification_action.action(response) message = notification_action.message logger.info(message.format(event=event)) event() if not notification_action.success: # processing failed # Return False if the message is to be retried return not dispatch.on_failure_retry return True
def create_assets(collection_payload: dict, order_payload: dict) -> group: """Create all assets in Alexandria if the do not exists. :param collection_payload: payload of order collection from briefy.alexandria :param order_payload: payload of order from leica :return: list of orders returned from listing payload """ factory = getUtility(IRemoteRestEndpoint) library_api = factory(config.ALEXANDRIA_BASE, 'collections', 'Collections') order = Objectify(order_payload) tasks = [] if order.requirement_items: for item in order.requirement_items: folder_contents = gdrive.folder_contents.delay( item.folder_id).get() images = folder_contents.get('images') collection_payload = library_api.get(item.id) image_tasks = [ chain( add_or_update_asset.s(image, collection_payload), s3.download_and_upload_file.s(image), ) for image in images ] tasks.extend(image_tasks) else: folder_contents = gdrive.folder_contents.delay(order.delivery.gdrive, extract_id=True).get() images = folder_contents.get('images') sub_folders = [ folder for folder in folder_contents.get('folders') if folder.get('name').lower().strip() in FOLDER_NAMES ] # make sure que get images also from sub folders for folder in sub_folders: images.extend(folder.get('images')) image_tasks = [ chain( add_or_update_asset.s(image, collection_payload), s3.download_and_upload_file.s(image), ) for image in images ] tasks.extend(image_tasks) return group(tasks)
def settings(self, value: t.Union[Objectify, t.Mapping]): """Project settings. Set all settings for a project. :value: Dictionary with all settings for a project. """ # 'PUT' semmantics setter. This will destroy everything on the way. # to change a single sub-field, consider changing the just the desired # entry along with a call to # sqlalchemy.orm.attributes.flag_modified(obj, data_field_name) # (check the correct underlying field_name on the settings.getter # above while we are in this transitional stage) value = Objectify(value, sentinel=None) self.tech_requirements = value.tech_requirements or {} self.delivery = value.delivery_config or {} self.add_order_roles = value.permissions.add_order or [] self.cancellation_window = value.dates.cancellation_window or 0 self.availability_window = value.dates.availability_window or 0 self.approval_window = value.dates.approval_window or 0
def download_and_upload_file(destiny: t.Tuple[str, str], image_payload: dict) -> str: """Download from GDrive and upload file to S3 bucket. :param destiny: tuple composed of (directory, file_name) :param image_payload: google drive file id :return: return the file_path """ directory, file_name = destiny image = Objectify(image_payload) if not file_exists(destiny): if not os.path.exists(directory): os.makedirs(directory) file_path = f'{directory}/{file_name}' with open(file_path, 'wb') as data: data.write(api.get_file(image.id)) result = upload_file(destiny) os.remove(file_path) else: result = f'{config.AWS_ASSETS_SOURCE}/{file_name}' return result
def process_message(self, message: SQSMessage) -> bool: """Process a message retrieved from the input_queue. :param message: A message from the queue :returns: Status from the process """ status = True body = message.body logger.info('Deis-worker: Processing message {0}'.format(body.get('id', None))) assignment = Objectify(body.get('data', {})) event = body.get('event_name', '') dispatch = Objectify(MESSAGE_DISPATCH.get(event, {})) if not dispatch: logger.info('Unknown event type - message {0} ignored'.format(body['id'])) return False try: status, payload = dispatch.action(assignment, self.session) except Exception as error: logger.error( 'Unknown exception raised on \'{0}\' assignment {1}. Error: {2}'.format( dispatch.name, assignment.dct, error ) ) raise # Let newrelic deal with it. event = None if status and dispatch.success_notification: event = dispatch.success_notification(payload) elif not status and dispatch.failure_notification: event = dispatch.failure_notification(payload) elif not status: logger.warning('Could not process message, and there are no ' 'further actions on failure: {0}'.format(body)) if event: event() return status
def transform(self, record: dict) -> dict: """Transform data payload from tsv file to Order payload.""" record = Objectify(record) project = self.project order_id = uuid.uuid4() number_required_assets = record._get('number_required_assets', 10) # parse first and last name names = record.full_name.split(' ') first_name = names[0] if len(names) > 1: last_name = ' '.join(names[0:]) else: last_name = '' # parse country code based on the country name country = pycountry.countries.get(name=record.country.strip(' ')) if not country: print(f'Country not found {record.country}') country = country.alpha_2 if country else project.country formatted_address = f'{record.address}, {record.district}, {record.locality}, ' \ f'{record.postal_code}, {project.country}' new_data = { 'id': order_id, 'project_id': project.id, 'asset_types': ['Image'], 'category': 'accommodation', 'description': '', 'title': record.title, 'requirements': '', 'current_type': 'order', 'state': 'accepted', 'number_required_assets': number_required_assets, 'source': 'briefy', 'customer_id': project.customer_id, 'customer_order_id': record.customer_order_id, 'price': 10000, # TODO: use from project if exists 'delivery': { 'gdrive': record._get('download_link', '') }, 'price_currency': 'EUR', # TODO: use google maps api to get the geohash 'location': { 'id': uuid.uuid4(), 'order_id': order_id, 'state': 'created', 'locality': record.locality, 'country': country, 'email': record.email, 'formatted_address': formatted_address, 'coordinates': [0, 0], 'additional_phone': None, 'timezone': country_timezones.get(country, [project.timezone])[0], 'first_name': first_name, 'last_name': last_name, 'mobile': record.mobile, 'info': { 'additional_info': '', 'province': record.locality, 'locality': record.locality, 'sublocality': record.district, 'route': record.address, 'street_number': '', 'country': project.country, 'postal_code': record.postal_code } }, } return new_data
def test_json_can_serialize_objectify(): from briefy.common.utils.data import Objectify a = [Objectify({'b': 1})] b = json.dumps(a) assert json.loads(b) == [{'b': 1}]
def ts_objectify(val: Objectify) -> str: """Serialize LabeledEnum instance to string.""" return val._get()
def add_or_update_asset(image_payload: dict, collection_payload: dict) -> t.Tuple[str, str]: """Add one assets in Alexandria if it do not exists. :param image_payload: image payload from briefy.gdrive :param collection_payload: payload of order collection from briefy.alexandria :return: asset file_path """ collection = Objectify(collection_payload) factory = getUtility(IRemoteRestEndpoint) library_api = factory(config.ALEXANDRIA_BASE, 'assets', 'Assets') image = Objectify(image_payload) data = library_api.query({'slug': image.id})['data'] if image.mimeType == 'image/jpeg': extension = 'jpg' elif len(image.name) >= 3: extension = image.name[-3:] else: extension = 'none' if not data: tags = ['gdrive', 'image'] tags.extend(collection.tags) asset_id = uuid.uuid4() file_name = f'{asset_id}.{extension}' source_path = f'{config.AWS_ASSETS_SOURCE}/{file_name}' payload = { 'slug': image.id, 'id': uuid.uuid4(), 'title': image.name, 'description': '', 'content_type': image.mimeType, 'source_path': source_path, 'tags': tags, 'collections': [collection.id], 'size': image.size, 'properties': { 'metadata': image.imageMediaMetadata, 'external_links': { 'view': image.webViewLink, 'download': image.webContentLink } } } data = library_api.post(payload) else: data = data[0] asset_id = data.get('id') data = library_api.get(asset_id) asset_collections = data.get('collections') if collection.id not in asset_collections: asset_collections.append(collection.id) data = library_api.put(asset_id, data) file_name = f'{asset_id}.{extension}' if not data: raise RuntimeError(f'Failed to add or update asset: {image_payload}') # in this case we should have one more directory if collection.content_type == 'application/collection.leica-order.requirement': order_id = collection.parent_id directory = f'{config.TMP_PATH}/{order_id}/{collection.id}' else: order_id = collection.id directory = f'{config.TMP_PATH}/{order_id}' logger.info( f'Asset added to alexandria. Path to save file: {directory}/{file_name}' ) return directory, file_name