def settings(self) -> Objectify:
"""Project settings.
Aggregate settings information about a project.
:return: Dictionary with all settings for a project.
"""
# TODO: These settings are in a transitional state while
# we move other configuration-related fields here.
# To preserve backwards compability, we simply proxy those
# fields - but their use should be deprecated as possible.
# (NB. Even with Objectify, there is no provision
# for write-back any of the "dates" subfields yet)
return Objectify({
'tech_requirements': self.tech_requirements,
'delivery_config': self.delivery,
'dates': {
'cancellation_window': self.cancellation_window,
'availability_window': self.availability_window,
'approval_window': self.approval_window,
},
'permissions': {
'add_order': self.add_order_roles
},
'order_type': self.order_type,
'project_type': self.project_type
})
def create_collections(self, order_payload: dict) -> dict:
"""Create all collections in Alexandria if the do not exists.
:param self: reference to the task class instance
:param order_payload: payload of order from leica
:return: order collection payload from the library
"""
factory = getUtility(IRemoteRestEndpoint)
library_api = factory(config.ALEXANDRIA_BASE, 'collections', 'Collections')
order = Objectify(order_payload)
collections = [
(order.customer, 'customer'),
(order.project, 'project'),
(order, 'order'),
]
parent_id = config.ALEXANDRIA_LEICA_ROOT
for item, type_ in collections:
result = library_api.get(item.id)
if not result:
payload = {
'slug': item.slug,
'id': item.id,
'title': item.title,
'description': item.description,
'content_type': f'application/collection.leica-{type_}',
'parent_id': parent_id,
'tags': [type_]
}
result = library_api.post(payload)
# this will be the new parent
parent_id = result.get('id')
if order.requirement_items:
for i, item in enumerate(order.requirement_items):
result = library_api.get(item.id)
if not result:
category_name = item._get('name', f'ItemName-{i}')
payload = {
'slug': slugify(category_name),
'id': item.id,
'title': item.category,
'description': item._get('description', ''),
'content_type':
'application/collection.leica-order.requirement',
'parent_id': order.id,
'tags': item.tags,
'properties': {
'gdrive': {
'folder_id': item.folder_id,
'parent_folder_id': item.parent_folder_id,
'created_by':
'9df18a79-44dc-4c2f-86aa-09bf7706ae86'
}
}
}
library_api.post(payload)
return library_api.get(order.id)
def main():
"""Execute import script."""
for item in PROJECTS_TO_IMPORT:
if not item.get('imported', False):
session = db_configure(Session)
importer = OrderImporter(item.get('file_name'),
Objectify(item.get('project')), session)
with transaction.manager:
importer()
def process_message(self, message: SQSMessage) -> bool:
"""Process a message retrieved from the input_queue.
:param message: A message from the queue
:returns: Status from the process
"""
body = message.body
data = Objectify(body.get('data', {}))
event = body.get('event_name', '')
message_id = body.get('id', '')
data.sqs_message_id = message_id
dispatch = Objectify(self.dispatch_map.get(event, {}))
if not dispatch:
logger.info('Unknown event type - message {0} ignored'.format(
body['id']))
return True
logger.info('Processing event {event}'.format(event=event))
try:
status, payload = dispatch.action(data)
except Exception as error:
msg = 'Unknown exception raised on \'{0}\' assignment {1}. \n' \
'Error: {2} \n Payload: {3}'
logger.error(msg.format(dispatch.name, data.id, error, data._dct))
raise # Let newrelic deal with it.
response = ResponseWrapper(data, payload)
notification_action = dispatch.notification_actions[status]
event = notification_action.action(response)
message = notification_action.message
logger.info(message.format(event=event))
event()
if not notification_action.success: # processing failed
# Return False if the message is to be retried
return not dispatch.on_failure_retry
return True
def create_assets(collection_payload: dict, order_payload: dict) -> group:
"""Create all assets in Alexandria if the do not exists.
:param collection_payload: payload of order collection from briefy.alexandria
:param order_payload: payload of order from leica
:return: list of orders returned from listing payload
"""
factory = getUtility(IRemoteRestEndpoint)
library_api = factory(config.ALEXANDRIA_BASE, 'collections', 'Collections')
order = Objectify(order_payload)
tasks = []
if order.requirement_items:
for item in order.requirement_items:
folder_contents = gdrive.folder_contents.delay(
item.folder_id).get()
images = folder_contents.get('images')
collection_payload = library_api.get(item.id)
image_tasks = [
chain(
add_or_update_asset.s(image, collection_payload),
s3.download_and_upload_file.s(image),
) for image in images
]
tasks.extend(image_tasks)
else:
folder_contents = gdrive.folder_contents.delay(order.delivery.gdrive,
extract_id=True).get()
images = folder_contents.get('images')
sub_folders = [
folder for folder in folder_contents.get('folders')
if folder.get('name').lower().strip() in FOLDER_NAMES
]
# make sure que get images also from sub folders
for folder in sub_folders:
images.extend(folder.get('images'))
image_tasks = [
chain(
add_or_update_asset.s(image, collection_payload),
s3.download_and_upload_file.s(image),
) for image in images
]
tasks.extend(image_tasks)
return group(tasks)
def settings(self, value: t.Union[Objectify, t.Mapping]):
"""Project settings.
Set all settings for a project.
:value: Dictionary with all settings for a project.
"""
# 'PUT' semmantics setter. This will destroy everything on the way.
# to change a single sub-field, consider changing the just the desired
# entry along with a call to
# sqlalchemy.orm.attributes.flag_modified(obj, data_field_name)
# (check the correct underlying field_name on the settings.getter
# above while we are in this transitional stage)
value = Objectify(value, sentinel=None)
self.tech_requirements = value.tech_requirements or {}
self.delivery = value.delivery_config or {}
self.add_order_roles = value.permissions.add_order or []
self.cancellation_window = value.dates.cancellation_window or 0
self.availability_window = value.dates.availability_window or 0
self.approval_window = value.dates.approval_window or 0
def download_and_upload_file(destiny: t.Tuple[str, str],
image_payload: dict) -> str:
"""Download from GDrive and upload file to S3 bucket.
:param destiny: tuple composed of (directory, file_name)
:param image_payload: google drive file id
:return: return the file_path
"""
directory, file_name = destiny
image = Objectify(image_payload)
if not file_exists(destiny):
if not os.path.exists(directory):
os.makedirs(directory)
file_path = f'{directory}/{file_name}'
with open(file_path, 'wb') as data:
data.write(api.get_file(image.id))
result = upload_file(destiny)
os.remove(file_path)
else:
result = f'{config.AWS_ASSETS_SOURCE}/{file_name}'
return result
def process_message(self, message: SQSMessage) -> bool:
"""Process a message retrieved from the input_queue.
:param message: A message from the queue
:returns: Status from the process
"""
status = True
body = message.body
logger.info('Deis-worker: Processing message {0}'.format(body.get('id', None)))
assignment = Objectify(body.get('data', {}))
event = body.get('event_name', '')
dispatch = Objectify(MESSAGE_DISPATCH.get(event, {}))
if not dispatch:
logger.info('Unknown event type - message {0} ignored'.format(body['id']))
return False
try:
status, payload = dispatch.action(assignment, self.session)
except Exception as error:
logger.error(
'Unknown exception raised on \'{0}\' assignment {1}. Error: {2}'.format(
dispatch.name,
assignment.dct,
error
)
)
raise # Let newrelic deal with it.
event = None
if status and dispatch.success_notification:
event = dispatch.success_notification(payload)
elif not status and dispatch.failure_notification:
event = dispatch.failure_notification(payload)
elif not status:
logger.warning('Could not process message, and there are no '
'further actions on failure: {0}'.format(body))
if event:
event()
return status
def transform(self, record: dict) -> dict:
"""Transform data payload from tsv file to Order payload."""
record = Objectify(record)
project = self.project
order_id = uuid.uuid4()
number_required_assets = record._get('number_required_assets', 10)
# parse first and last name
names = record.full_name.split(' ')
first_name = names[0]
if len(names) > 1:
last_name = ' '.join(names[0:])
else:
last_name = ''
# parse country code based on the country name
country = pycountry.countries.get(name=record.country.strip(' '))
if not country:
print(f'Country not found {record.country}')
country = country.alpha_2 if country else project.country
formatted_address = f'{record.address}, {record.district}, {record.locality}, ' \
f'{record.postal_code}, {project.country}'
new_data = {
'id': order_id,
'project_id': project.id,
'asset_types': ['Image'],
'category': 'accommodation',
'description': '',
'title': record.title,
'requirements': '',
'current_type': 'order',
'state': 'accepted',
'number_required_assets': number_required_assets,
'source': 'briefy',
'customer_id': project.customer_id,
'customer_order_id': record.customer_order_id,
'price': 10000, # TODO: use from project if exists
'delivery': {
'gdrive': record._get('download_link', '')
},
'price_currency': 'EUR',
# TODO: use google maps api to get the geohash
'location': {
'id': uuid.uuid4(),
'order_id': order_id,
'state': 'created',
'locality': record.locality,
'country': country,
'email': record.email,
'formatted_address': formatted_address,
'coordinates': [0, 0],
'additional_phone': None,
'timezone': country_timezones.get(country,
[project.timezone])[0],
'first_name': first_name,
'last_name': last_name,
'mobile': record.mobile,
'info': {
'additional_info': '',
'province': record.locality,
'locality': record.locality,
'sublocality': record.district,
'route': record.address,
'street_number': '',
'country': project.country,
'postal_code': record.postal_code
}
},
}
return new_data
def test_json_can_serialize_objectify():
from briefy.common.utils.data import Objectify
a = [Objectify({'b': 1})]
b = json.dumps(a)
assert json.loads(b) == [{'b': 1}]
def ts_objectify(val: Objectify) -> str:
"""Serialize LabeledEnum instance to string."""
return val._get()
def add_or_update_asset(image_payload: dict,
collection_payload: dict) -> t.Tuple[str, str]:
"""Add one assets in Alexandria if it do not exists.
:param image_payload: image payload from briefy.gdrive
:param collection_payload: payload of order collection from briefy.alexandria
:return: asset file_path
"""
collection = Objectify(collection_payload)
factory = getUtility(IRemoteRestEndpoint)
library_api = factory(config.ALEXANDRIA_BASE, 'assets', 'Assets')
image = Objectify(image_payload)
data = library_api.query({'slug': image.id})['data']
if image.mimeType == 'image/jpeg':
extension = 'jpg'
elif len(image.name) >= 3:
extension = image.name[-3:]
else:
extension = 'none'
if not data:
tags = ['gdrive', 'image']
tags.extend(collection.tags)
asset_id = uuid.uuid4()
file_name = f'{asset_id}.{extension}'
source_path = f'{config.AWS_ASSETS_SOURCE}/{file_name}'
payload = {
'slug': image.id,
'id': uuid.uuid4(),
'title': image.name,
'description': '',
'content_type': image.mimeType,
'source_path': source_path,
'tags': tags,
'collections': [collection.id],
'size': image.size,
'properties': {
'metadata': image.imageMediaMetadata,
'external_links': {
'view': image.webViewLink,
'download': image.webContentLink
}
}
}
data = library_api.post(payload)
else:
data = data[0]
asset_id = data.get('id')
data = library_api.get(asset_id)
asset_collections = data.get('collections')
if collection.id not in asset_collections:
asset_collections.append(collection.id)
data = library_api.put(asset_id, data)
file_name = f'{asset_id}.{extension}'
if not data:
raise RuntimeError(f'Failed to add or update asset: {image_payload}')
# in this case we should have one more directory
if collection.content_type == 'application/collection.leica-order.requirement':
order_id = collection.parent_id
directory = f'{config.TMP_PATH}/{order_id}/{collection.id}'
else:
order_id = collection.id
directory = f'{config.TMP_PATH}/{order_id}'
logger.info(
f'Asset added to alexandria. Path to save file: {directory}/{file_name}'
)
return directory, file_name