class Postman:
DEFAULT_FEATURED_MEDIA = 151
DEFAULT_CATEGORY = 13
def __init__(self, url, username, password):
self.__wp_api = API(url=url,
api="wp-json",
version='wp/v2',
wp_user=username,
wp_pass=password,
basic_auth=True,
user_auth=True,
consumer_key="",
consumer_secret="")
assert self.__wp_api.get("categories").status_code == 200
def upload_media(self, file_path):
assert os.path.exists(file_path), "img should exist at {}".format(
file_path)
data = open(file_path, 'rb').read()
filename = os.path.basename(file_path)
_, extension = os.path.splitext(filename)
headers = {
'cache-control': 'no-cache',
'content-disposition': 'attachment; filename=%s' % filename,
'content-type': 'image/%s' % extension
}
response = self.__wp_api.post("media", data, headers=headers)
print "Media upload response status:{}".format(response.status_code)
return response.json()
def create_post(self, post_dir, status):
assert os.path.exists(
post_dir), "Post Directory should exist at {}".format(post_dir)
with open(os.path.join(post_dir, "metadata.json"), 'r') as f:
metadata = json.loads(f.read())
post = {
'slug': metadata['information']['slug'],
'title': metadata['information']['title']
}
if 'title-img' in metadata['information'] and metadata[
'information']['title-img']:
media = self.upload_media(
os.path.join(post_dir,
metadata['information']['title-img']))
post['featured_media'] = media['id']
else:
post['featured_media'] = Postman.DEFAULT_FEATURED_MEDIA
post['status'] = status
post['categories'] = [Postman.DEFAULT_CATEGORY]
assert metadata['information']['content-file']
with open(
os.path.join(
post_dir,
metadata['information']['content-file'])) as c:
post['content'] = c.read().decode('utf-8')
headers = {'content-type': 'application/json'}
response = self.__wp_api.post("posts", data=post, headers=headers)
print "Create Post response status:{}".format(response.status_code)
return response
class WooCommerceShim(Database):
"""
Contains various methods for interacting with
the WooCommerce API. These methods will do things
such as adding new products, and removing sold
products.
"""
def __init__(self, *args, **kwargs):
super(WooCommerceShim, self).__init__(*args, **kwargs)
# Setup logging
self.log = logging.getLogger(__name__)
self.log.setLevel(os.environ.get('log_level', 'INFO'))
self.log.addHandler(LOG_HANDLER)
self.api = WCAPI(url=os.environ.get('woo_url', False),
consumer_key=os.environ.get('woo_key', False),
consumer_secret=os.environ.get('woo_secret', False))
self.wp_api = WPAPI(url=os.environ.get('woo_url', False),
api='wp-json',
version='wp/v2',
wp_user=os.environ.get('wordpress_user', False),
wp_pass=os.environ.get('wordpress_app_password',
False),
basic_auth=True,
user_auth=True,
consumer_key=False,
consumer_secret=False)
mapping_path = os.environ.get(
'category_mapping',
'database/ebay-to-woo-commerce-category-map.json')
try:
with open(mapping_path, 'r') as mapping_file:
self.category_mapping = json.load(mapping_file)
except IOError:
self.category_mapping = None
def __does_image_exist_on_woocommerce(self, slug):
"""
Searches the Wordpress media library for
any files that have a URL `slug` that
matches the one provided
Returns True if the file exists, and False otherwise
This method is unreliable! Duplicates are basically guarenteed to happen...
"""
self.log.info('Checking if a file has a slug matching: %s' % (slug))
result = self.wp_api.get('/media?slug=%s' % (slug)).json()
if len(result) == 0:
return False
return True
def __divide_into_chunks(self, iterable, chunk_size=100):
"""
Used to make bulk requests via the API, which limits
the amount of products to change at once to 100
`iterable` is something that can be iterated over, be
it a list or a range. When a range, you must wrap the
output of this method in `list()`
`chunk_size` is optional, and defines how many products
to change per request. The default of 100 is the maximum
that the API will allow
Returns the `iterable` containing as many items as are
in the `chunk_size`
"""
for i in range(0, len(iterable), chunk_size):
yield iterable[i:i + chunk_size]
def __search_map(self, value, field):
"""
Uses List Comprehension to search for the `value` in the `field`.
Normal usage would be similar to `self.__search_map(ebay_category_id, 'ebay_ids')`
Returns an integer, which is the first matching Woo Commerce ID for
the selected `value` (in the case that one ebay category is mapped to
multiple woo commerce categories)
When a matching category can't be found, this method will call itself
to search for the "Uncategorized" `value` on the "wc-name" `field`
"""
try:
mapped = [
key for key in self.category_mapping if value in key[field]
]
return int(mapped[0]['wc-id'])
except IndexError: # Couldn't find it, return the uncategorized id
return self.__search_map('Uncategorized', 'wc-name')
def does_product_exist(self, item_id):
"""
Determines if the product with the `item_id` has
already been uploaded to WooCommerce, by checking
for the truthyness of `post_id`
"""
data = self.db_get_product_data(item_id)
if data.get('post_id') is not None:
return True
return False
def get_mapped_category_id(self, ebay_category_id):
"""
Determines if the user provided a category mapping, and if so
returns an integer, which is the Woo Commerce category id that
is mapped to the `ebay_category_id` (or the Uncategorized category
id if a mapping can not be found)
In the case that the user has not provided a category mapping,
this method returns None
"""
if self.category_mapping is not None:
return self.__search_map(ebay_category_id, 'ebay_ids')
return None
def download_product_images_from_ebay(self, item_id):
"""
Downloads all of the images for a provided `item_id` and
returns a dictionary containing the image name, mime type, and
bytes-like object for the raw images
The image URLs come from the database table `item_metadata`,
which is populated when `self.__get_item_metadata()` runs
"""
count = 0
return_images = list()
image_urls = self.db_get_product_image_urls(item_id)
image_urls_count = len(image_urls)
if image_urls_count > 0:
self.log.info("Found %d image URLs for: %s" %
(image_urls_count, item_id))
for image in image_urls:
url = image.get('value', '')
if image.get('post_id') is not None:
self.log.warning(
"We've already uploaded %s, skipping download" % (url))
continue
self.log.info("Downloading %s" % (url))
req = requests.get(url)
if req.content:
mime_type = req.headers.get('Content-Type', '')
slug = '%s-%d' % (item_id, count)
extension = mime_type.split('/')[1]
filename = '%s.%s' % (slug, extension)
if 'image' not in mime_type:
msg = "%d didn't get an image somehow. Content type was: %s"
self.log.error(msg % (item_id, mime_type))
continue
return_images.append(
Image(slug=slug,
ebay_url=url,
name=filename,
mime_type=mime_type,
data=req.content))
# self.log.info("Image %s downloaded" % (filename))
if count < image_urls_count:
self.log.debug(
"Waiting a quarter second until next download")
time.sleep(0.25)
else:
self.log.error(
"No content returned. Is %s reachable in a browser?" %
(url))
count += 1
else:
self.log.warning("No Image URLs found for item: %s" % (item_id))
return return_images
def upload_image_to_woocommerce(self, image, post_id):
"""
Uploads the provided `image` to wordpress, and returns the response
`image` is a dictionary containing the following keys:
`name` - This is the destination file name, including extension
`type` - This is the MIMETYPE of the image, usually derived from the extension
`data` - This is a bytes-like object representing the entire image. We get this
from dowloading an image directly from Ebay's servers and temporarily storing it
in memory
`post_id` is the post in which to attach the image to. This is returned in the
response from `self.create_product()`
Returns either a string containing the URL the image can be found at, or False
if the image fails to be uploaded
"""
self.log.info("Uploading %s to wordpress" % (image.name))
endpoint = '/media?post=%d' % (post_id)
headers = {
'cache-control': 'no-cache',
'content-disposition': 'attachment; filename=%s' % (image.name),
'content-type': '%s' % (image.mime_type)
}
# Don't upload a duplicate image if it was uploaded in the past
if self.__does_image_exist_on_woocommerce(image.slug):
self.log.warning(
"Image %s already exists on wordpress. Not uploading again" %
(image.name))
return None, None
# Upload the image
response = self.wp_api.post(endpoint, image.data, headers=headers)
try:
image_id = response.json().get('id')
url = response.json().get('guid', dict).get('raw')
self.log.debug("Uploaded %s to %s" % (image.name, url))
return image_id, url
except AttributeError:
self.log.error('Could not upload %s' % image.name)
return None, None
def upload_product_images(self, item_id):
"""
With the provided `item_id`, the database
is searched for the post id (set during
`create_product`)
When the post_id is found, it will be used
to download the images for that product
from ebay, and then upload the images
"""
post_id = self.db_woo_get_post_id(item_id)
gallery = []
if post_id is not None:
for image in self.download_product_images_from_ebay(item_id):
image_id, url = self.upload_image_to_woocommerce(
image, post_id)
if image_id and url:
self.db_metadata_uploaded(image_id, item_id)
gallery.append({'id': image_id})
# Add the images to the gallery
self.api.put('products/%d' % (post_id), {'images': gallery}).json()
else:
self.log.warning('The product %d has not yet been uploaded' %
(item_id))
return self
def create_product(self, item_id):
"""
Pulls the product related to the `item_id`
out of the database and uploads it to WooCommerce
Returns the result as JSON
"""
attributes = list()
attributes_to_upload = list()
self.log.info('Creating a WooCommerce product from ebay id: %s' %
(item_id))
if self.does_product_exist(item_id):
self.log.warning(
'Product with item id %d already exists, skipping' % (item_id))
return self
product = self.db_get_product_data(item_id)
attributes = self.db_get_all_product_metadata(item_id)
# Strip out any pictures
attributes = [
attribute for attribute in attributes
if attribute['key'] != 'picture_url'
]
# Format the attributes in a way that WooCommerce is expecting
for index, attribute in enumerate(attributes):
attributes_to_upload.append({
'name': attribute['key'],
'options': [attribute['value']],
'visible': True,
'variation': True,
'position': index,
})
upload_data = {
'name': product['title'],
'type': 'simple',
'status': 'publish',
'short_description': product['condition_description'],
'description': DEFAULT_DESCRIPTION,
'sku': product['sku'],
# 'attributes': attributes_to_upload,
# 'default_attributes': attributes_to_upload,
}
# Add the category id
category_id = self.get_mapped_category_id(product.get(
'category_id', 0))
if category_id is not None:
upload_data['categories'] = [{'id': category_id}]
res = self.api.post('products', upload_data).json()
self.log.debug(res)
if res.get('id', False):
self.db_product_uploaded(res['id'], item_id)
else:
# Invalid or duplicate sku
if res.get('code') == 'product_invalid_sku':
if res.get('data') and res['data'].get('resource_id'):
new_post_id = res['data']['resource_id']
self.log.warning(
'The SKU for %s already exists for %s. Updating.' %
(new_post_id, item_id))
self.db_product_uploaded(new_post_id, item_id)
else:
self.log.error('Unable to retrive product_id')
self.log.debug(res)
self.log.debug(upload_data)
return self
def delete_product_images(self, item_id):
"""
With the provided `item_id`, an API request will
be made to WooCommerce to identify all images
associted with it. Then, it will delete each of
those images.
"""
# This feature has not been implemented.
# Delete media through wordpress directly
pass
def delete_product(self, item_id):
"""
With the provided `item_id`, an API request will
be made to WooCommerce to force delete the item
The `item_id` is supplied by the queue, which gets
them from `db.db_get_inactive_uploaded_item_ids()`
When an item is force deleted, it will not appear
in the "Trash"
Returns the response as a dictionary or None if
there is no post id
"""
post_id = self.db_woo_get_post_id(item_id)
if post_id is not None:
self.log.info('Deleting %d from WooCommerce' % (item_id))
try:
response = self.api.delete('products/%d' % (post_id),
params={
'force': True
}).json()
except TypeError:
self.log.error("Got unexpected response type: %s" %
(str(response)))
return None
self.delete_product_images(post_id)
status_code = response.get('data', dict).get('staus', 500)
if status_code == 404:
self.log.warning("Product was already deleted")
elif status_code < 300 and status_code > 199:
self.log.info('Product deleted')
else:
self.log.debug(response)
return response
return None
def delete_all_products_in_range(self, id_range, chunk_size=100):
"""
With a provided `id_range`, which is expected to be
a `range` or `list` type, multiple bulk requests
will be made to the Woo Commerce API to delete
those items.
When `id_range` is of type(range), your ending ID needs
to be the last ID to delete + 1
Returns None
"""
self.log.info('Deleting products from %d to %d' %
(id_range[0], id_range[-1]))
# The API says that it supports chunks up to 100 items, but in testing
# it would always time out, even if it successfully deleted the items
# with any chunk size greater than or equal to 50
for chunk in self.__divide_into_chunks(id_range, chunk_size):
post_ids = list(chunk)
data = {'delete': post_ids}
self.api.post('products/batch', data)
self.log.info('Deleted ids %s' % (post_ids))
for post_id in post_ids:
self.delete_product_images(post_id)
def try_command(self, command, data):
"""
Wrapper for running methods.
Verifies that we support the method, raising a NameError if not
and then runs the method specified in the `command` argument in
a try, except statement
`command` is a string that is inside `__available_commands`
`data` is dependent on the type of command that is being ran.
In most instances, it is an integer containing the ebay ItemID.
With the `delete_all_products` command, it is either a range or
a list containing the post ids for existing products
"""
__available_commands = [
'create_product',
'delete_product',
'upload_images',
'delete_all_products',
]
err_msg = "Command %s is unrecognized. Supported commands are: %s" % (
command, ', '.join(__available_commands))
if command not in __available_commands:
self.log.exception(err_msg)
raise NameError(err_msg)
try:
if command == 'create_product':
return self.create_product(data)
elif command == 'delete_product':
return self.delete_product(data)
elif command == 'upload_images':
return self.upload_product_images(data)
elif command == 'delete_all_products':
return self.delete_all_products_in_range(data)
else:
self.log.exception(err_msg)
raise NameError(err_msg)
# The several kinds of timeout exceptions that are normally returned by the API
except (timeout, ReadTimeoutError, requests.exceptions.ConnectTimeout,
requests.exceptions.ReadTimeout):
self.log.warning(
'The Previous request Timed Out. Waiting 5s before retrying')
time.sleep(5)
self.try_command(command, data)
