def main(website,depth):
website = "https://" + website
c = Crawler(website,int(depth))
c.crawl()
c.save_to_json()
index = json.load(open('index.json','r'))
for ind,elem in enumerate(index):
i14yClient.create(ind,elem['content'],elem['path'],
elem['created'],os.environ["drawer_handle"],os.environ["search_secret_token"],
title=elem['title'],description=elem['description'],
promote=elem['promote'],language=elem['language'])
from ingestion.engine import Crawler
from api.clients import i14yClient
from sys import argv
import pickle
import json
website_creds = pickle.load(open("website_creds.pickle","r"))
backend_creds = pickle.load(open("backend_creds.pickle","r"))
c = Crawler(argv[1],int(argv[2])) #,username=website_creds["username"],password=website_creds["password"],basic_auth_required=True)
c.crawl()
#print c.data
#print c.urls
c.save_to_json()
index = json.load(open('index.json','r'))
for ind,elem in enumerate(index):
i14yClient.create(ind,elem['content'],elem['path'],
elem['created'],backend_creds["drawer_handle"],backend_creds["secret_token"],
title=elem['title'],description=elem['description'],
promote=elem['promote'],language=elem['language'])
