def ensure_data_directory(relative_path: Path = None) -> Path:
"""Checks if a directory in the data dir path exists. Creates it if necessary
Args:
relative_path: A Path object pointing to a file relative to the data directory
Returns:
The absolute path Path object
"""
if relative_path is None:
return Path(config.data_dir())
try:
path = Path(config.data_dir(), relative_path)
# if path points to a file, create parent directory instead
if path.suffix:
if not path.parent.exists():
path.parent.mkdir(exist_ok=True, parents=True)
else:
if not path.exists():
path.mkdir(exist_ok=True, parents=True)
return path
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
def download_shopify_resource(shopify_resource: ShopifyResource):
"""Downloads data for a single Shopify resource
https://help.shopify.com/api/reference
Args:
shopify_resource: a Shopify resource e.g Customer, Order, Product
"""
data = []
page = 1
while True:
resource = shopify_resource.find(limit=250,
page=page,
status=config.order_status())
if len(resource) > 0:
data.extend(resource)
page += 1
else:
relative_filepath = Path(
config.data_dir(), '{resource_name}-{version}.json.gz'.format(
resource_name=shopify_resource.plural,
version=OUTPUT_FILE_VERSION))
filepath = ensure_data_directory(relative_filepath)
with tempfile.TemporaryDirectory() as tmp_dir:
tmp_filepath = Path(tmp_dir, relative_filepath)
tmp_filepath.parent.mkdir(exist_ok=True, parents=True)
with gzip.open(str(tmp_filepath),
'wt') as tmp_ad_performance_file:
tmp_ad_performance_file.write(
json.dumps([c.to_dict() for c in data]))
shutil.move(str(tmp_filepath), str(filepath))
break
def download_shopify_resource(shopify_resource: ShopifyResource):
"""Downloads data for a single Shopify resource
https://help.shopify.com/api/reference
Args:
shopify_resource: a Shopify resource e.g Customer, Order, Product
"""
from time import sleep
data = []
page = 1
previous_page = 0
while True:
number_of_attempts = 0
while True:
try:
resource = shopify_resource.find(limit=250,
page=page,
status=config.order_status())
if len(resource) > 0 or (previous_page == page):
break
except:
if number_of_attempts < 5:
duration = 4 * number_of_attempts + 1
logging.info(
'Loading shopify data retry #{attempt} in {duration} seconds'
.format(attempt=number_of_attempts, duration=duration))
sleep(duration)
number_of_attempts += 1
else:
raise
previous_page = page
if len(resource) > 0:
previous_page = page
data.extend(resource)
page += 1
else:
relative_filepath = Path(
config.data_dir(), '{resource_name}-{version}.json.gz'.format(
resource_name=shopify_resource.plural,
version=OUTPUT_FILE_VERSION))
filepath = ensure_data_directory(relative_filepath)
with tempfile.TemporaryDirectory() as tmp_dir:
tmp_filepath = Path(tmp_dir, relative_filepath)
tmp_filepath.parent.mkdir(exist_ok=True, parents=True)
with gzip.open(str(tmp_filepath),
'wt') as tmp_ad_performance_file:
tmp_ad_performance_file.write(
json.dumps([c.to_dict() for c in data]))
shutil.move(str(tmp_filepath), str(filepath))
break