def timed_job():
print('This job is run every three minutes.')
report_type = '_GET_MERCHANT_LISTINGS_DATA_'
users = firebase_api.query_objects('users')
for sellerid, user in users.iteritems():
authToken = user['authToken']
markets = user['marketplaces']
for marketname, marketplaceid in markets.iteritems():
result = mws_reports.run_report_pipeline(sellerid, authToken, report_type, marketplaceid)
if result != None:
for data in result:
if data != None and data['seller_sku'] != None:
sku = data['seller_sku'].replace('.', '*');
if data['item_name'] != None:
data['item_name'] = data['item_name'].decode('latin1')
if data['item_description'] != None:
data['item_description'] = data['item_description'].decode('latin1')
firebase_api.update_object('report/' + sellerid + '/' + marketplaceid + '/' + report_type + '/', sku, data)
else:
print "No report"
def get_commissions_for_user(username,
is_paid_out=None,
is_approved=None,
designer_username=None,
start_date=None,
end_date=None):
commissions = firebase_api.query_objects("commissions/" + username) or {}
filtered = {}
for key in commissions:
is_valid = True
if is_paid_out != None and is_paid_out and not commissions[key].get(
"paid_out"):
is_valid = False
elif is_paid_out != None and not is_paid_out and commissions[key].get(
"paid_out"):
is_valid = False
elif is_approved != None and is_approved and not commissions[key].get(
"approved"):
is_valid = False
elif is_approved != None and not is_approved and commissions[key].get(
"approved"):
is_valid = False
elif designer_username != None and designer_username != commissions[
key].get("designer_username"):
is_valid = False
elif start_date != None and start_date > commissions[key].get(
"created_at"):
is_valid = False
elif start_date != None and end_date < commissions[key].get(
"created_at"):
is_valid = False
if is_valid:
filtered[key] = commissions[key]
return filtered
def import_local_merch_data_into_db(datestr,
job_id,
force=False,
should_post=False):
download_merch_data_in_s3(datestr, job_id)
job_metadata = firebase_api.query_objects("scrape_merch_asin_task/" +
datestr + "/" + job_id)
print(job_metadata, "hello")
# Have this code logic so that we can do imports for files without jobs attached to them
if should_post and not job_metadata:
return
else:
job_metadata = {}
if not force and job_metadata.get("status") == "processed":
return
try:
data = open(job_id + "_data.csv", "r")
keywords = open(job_id + "_keywords.csv", "r")
except Exception as e:
print(e)
return False
headers = [
'asin', 'product_type_name', 'marketplace_id', 'title', 'color',
'search_index', 'brand', 'publisher', 'binding', 'label',
'parent_asin', 'part_number', 'department', 'studio', 'genre',
'discovery_keyword', 'product_group', 'image', 'browse_node',
'manufacturer'
]
data_reader = csv.reader(data)
for row in data_reader:
data_dict = {}
for i, header in enumerate(headers):
data_dict[header] = row[i]
if not data_dict.get("asin"):
continue
item = AsinMetadata(data=data_dict)
db.session.add(item)
try:
db.session.commit()
except Exception as e:
db.session.rollback()
print("already found the asin... skipping", item.id)
for row in keywords.readlines()[0:-1]:
keyword = row.strip('\n')
indexed_keyword = IndexedKeyword(data={"keyword": keyword})
db.session.add(indexed_keyword)
try:
db.session.commit()
except Exception as e:
db.session.rollback()
print("it rolled back")
if should_post == True:
firebase_api.update_object("scrape_merch_asin_task/" + datestr, job_id,
{"status": "processed"})
return True
import firebase_api
import mws_reports
import xml.etree.ElementTree as ET
import time
import inspect
import datetime
import json
import os
import csv
report_type = '_GET_FLAT_FILE_ORDERS_DATA_'
users = firebase_api.query_objects('users')
for sellerid, user in users.iteritems():
authToken = user['authToken']
markets = user['marketplaces']
for marketname, marketplaceid in markets.iteritems():
result = mws_reports.run_report_pipeline(sellerid, authToken, report_type, marketplaceid)
if result != None:
for data in result:
if data != None and data['sku'] != None:
sku = data['sku'].replace('.', '*');
orderid = data['order_id']
if data['product_name'] != None:
data['product_name'] = data['product_name'].decode('latin1')
if data['buyer_name'] != None:
def get_payout(username, designer_username, payout_id):
return firebase_api.query_objects("payouts/" + username + "/" +
designer_username + "/" + payout_id)
def get_assignments_for_user(username, status=None, designer_username=None):
assignments = firebase_api.query_objects("assignments/" + username) or {}
return filter_assignments(assignments, status, designer_username)
def get_commission(username, commission_id):
return firebase_api.query_objects("commissions/" + commission_id)
def get_assignment(username, assignment_id):
assignment = firebase_api.query_objects("assignments/" + username + "/" +
assignment_id)
assignment["id"] = assignment_id
return assignment
def get_assignments_dict_for_user(username):
assignments = firebase_api.query_objects("assignments/" + username) or {}
return assignments
def get_vas_for_user(username):
vas = firebase_api.query_objects("virtual_assistants/" + username) or {}
for key in vas:
vas[key]["designer_username"] = key
return vas.values()
def get_clients_for_va(designer_username):
clients = firebase_api.query_objects("clients/" + designer_username) or {}
for user_id in clients:
clients[user_id]["username"] = user_id
return clients.values()
expired_with_valid_auth_token.append(user)
valid_users = []
for user in expired_with_valid_auth_token:
#print("hihii")
data_collector = DataCollector(user)
product, error = data_collector.grab_data_for_asin("B0755LLJST")
if error and "Access to Products.GetMatchingProductForId is denied".lower(
) in error.get("errorMessage").lower():
# @TODO: Maybe cache these bad accounts somewhere for faster initial loading?
#print("remove that guy")
pass
elif error and "The seller does not have an eligible Amazon account to call Amazon MWS".lower(
) in error.get("errorMessage").lower():
#print("got em")
pass
elif error:
pass
print("f**k")
print(error)
else:
#print("valid")
valid_users.append(user)
#print(product)
return valid_users
valid_users = get_valid_users()
firebase_api.update_object("", "validInactiveMwsAccounts", valid_users)
print firebase_api.query_objects("validInactiveMwsAccounts")
import datetime
from datetime import timedelta
import firebase_api
from scrapers.asin_indexer import import_local_merch_data_into_db
q = Queue(connection=conn)
# iterate over last seven days and do any job that was missing
curr = datetime.datetime.today()
num_days = 7
i = 0
while i < num_days:
print("hi", i)
datestr = str(curr).split()[0]
jobs_dict = firebase_api.query_objects("scrape_merch_asin_task/" + datestr)
if not jobs_dict:
curr = curr - timedelta(days=1)
i += 1
continue
for job_id in jobs_dict:
if jobs_dict[job_id].get("status") != "uploaded":
continue
force = False
should_post = True
args = (datestr, job_id, force, should_post)
print(job_id, datestr)
q.enqueue_call(import_local_merch_data_into_db,
args=args,
timeout=3600)
curr = curr - timedelta(days=1)
import firebase_api
import mws_reports
from mws import mws
import xml.etree.ElementTree as ET
import time
ACCESS_KEY = 'AKIAJZ6R7P2H65YKTHLQ' #replace with your access key
SECRET_KEY = 'BDmbHcnfrWCB0i33bMN2YqW13UHG2ptV3s5HYqzs' #replace with your secret key
usersDict = firebase_api.query_objects("users")
reports = ["_GET_MERCHANT_LISTINGS_DATA_", "_GET_AFN_INVENTORY_DATA_", "_GET_FBA_REIMBURSEMENTS_DATA_", "_GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA_"]
progress_data = {}
pipeline_finished = False
while not pipeline_finished:
for objectId in usersDict:
found_unfinished_user = False
user = usersDict[objectId]
username = user["username"]
if "sellerId" not in user:
#print("Skipping " + username)
continue
seller_id = user["sellerId"]
auth_token = user["authToken"]
if username not in progress_data:
found_unfinished_user = True
progress_data[username] = {}
