def schedule_spider(project,
endpoint,
apikey,
spider,
arguments=(),
settings=(),
priority=DEFAULT_PRIORITY,
units=None,
tag=(),
environment=()):
client = ScrapinghubClient(apikey, dash_endpoint=endpoint)
try:
project = client.get_project(project)
args = dict(x.split('=', 1) for x in arguments)
cmd_args = args.pop('cmd_args', None)
meta = args.pop('meta', None)
job = project.jobs.run(
spider=spider,
meta=json.loads(meta) if meta else {},
cmd_args=cmd_args,
job_args=args,
job_settings=dict(x.split('=', 1) for x in settings),
priority=priority,
units=units,
add_tag=tag,
environment=dict(x.split('=', 1) for x in environment),
)
return job.key
except ScrapinghubAPIError as e:
raise RemoteErrorException(str(e))
def get_last_job_ids(self):
project_id = os.environ.get("SCRAPY_PROJECT_ID")
api_key = self.spider.settings.get("SCRAPINGHUB_API_KEY")
if not project_id or not api_key:
return []
client = ScrapinghubClient(api_key)
project = client.get_project(project_id)
jobs = project.jobs.list()
if not jobs:
return []
# find last job for spider searchterm same spider
# can be invoked with different searchterms
last_matching_job = None
for each in jobs:
key = each["key"]
job = client.get_job(key)
metadata = dict(job.metadata.list())
searchterm = metadata.get("spider_args", {}).get("searchterm", "")
if self.spider.searchterm == searchterm:
last_matching_job = job
break
if not last_matching_job:
return []
return [item["id"] for item in last_matching_job.items.iter()]
def __init__(self,
collection_name,
project_id=None,
apikey=None,
autodetect_partitions=True):
"""
collection_name - target collection
project_id - target project id
apikey - hubstorage apikey with access to given project. If None, delegate to scrapinghub lib.
autodetect_partitions - If provided, autodetect partitioned collection. By default is True. If you want instead to force to read a non-partitioned
collection when partitioned version also exists under the same name, use False.
"""
self.hsc = ScrapinghubClient(apikey)._hsclient
project_id = project_id or get_project_id()
self.hsp = self.hsc.get_project(project_id)
num_partitions = None
if autodetect_partitions:
num_partitions = get_num_partitions(self.hsp, collection_name)
if num_partitions:
log.info(
"Partitioned collection detected: %d total partitions.",
num_partitions)
self.collections = []
if num_partitions:
for p in range(num_partitions):
self.collections.append(
self.hsp.collections.new_store("{}_{}".format(
collection_name, p)))
else:
self.collections.append(
self.hsp.collections.new_store(collection_name))
def getDataXoso():
# Enter ScrapingHub
apikey = '40f9881d52794d7bb09b9f5ee6d12a3e' # your API key as a string
client = ScrapinghubClient(apikey)
projectID = 410647
project = client.get_project(projectID)
# get spider
spiderID = 'quotes'
spider = project.spiders.get(spiderID)
jobs_summary = spider.jobs.iter()
job_keys = [j['key'] for j in jobs_summary]
print(job_keys)
result = []
for job_key in job_keys:
job = project.jobs.get(job_key)
# Check to see if the job was completed
if job.metadata.get(u'close_reason') == u'finished':
for item in job.items.iter():
result.append(item)
return result
def create_json_schema(source_key: str,
item_numbers: List[int] = None) -> dict:
client = ScrapinghubClient()
if helpers.is_collection_key(source_key):
store = api.get_collection(source_key)
items_count = store.count()
elif helpers.is_job_key(source_key):
job = client.get_job(source_key)
items_count = api.get_items_count(job)
store = job.items
else:
logger.error(f"{source_key} is not a job or collection key")
return
if items_count == 0:
logger.error(f"{source_key} does not have any items")
return
item_n_err = "{} is a bad item number, choose numbers between 0 and {}"
if item_numbers:
item_numbers.sort()
if item_numbers[-1] >= items_count or item_numbers[0] < 0:
logger.error(item_n_err.format(item_numbers[-1], items_count - 1))
return
else:
item_numbers = set_item_no(items_count)
samples = []
for n in item_numbers:
items = api.get_items(source_key, start_index=n, count=1)
samples.append(items[0])
return infer_schema(samples)
def __init__(self):
self.workflow_loop_enabled = False
self.args = self.parse_args()
self.client = ScrapinghubClient(self.args.apikey)
self.project_id = resolve_project_id(self.args.project_id
or self.project_id)
if not self.project_id:
self.argparser.error('Project id not provided.')
def ready(self):
global test
apikey = '88133cc793ab4296b56db8a87eaae1ec'
client = ScrapinghubClient(apikey)
test = client.get_job('223795/1/3')
test = sorted(test.items.list(),
key=lambda k: k['score'],
reverse=True)
def __init__(self, auth, project_id, frontier, batch_size=0):
self._client = ScrapinghubClient(auth=auth)
self._hcf = self._client.get_project(project_id).frontiers
self._frontier = self._hcf.get(frontier)
self._links_count = defaultdict(int)
self._links_to_flush_count = defaultdict(int)
self._batch_size = batch_size
self._hcf_retries = 10
def __init__(self):
self.apikey = '' # your API key as a string
self.client = ScrapinghubClient(self.apikey)
self.project_num = 0
self.project = self.client.get_project(self.project_num)
self.neighborhood_spider = self.get_neighborhood_spider()
self.listing_spider = self.get_listing_spider()
self.airdna_spider = self.get_airdna_spider()
def __init__(self, input_uri, settings):
super().__init__(settings)
client = ScrapinghubClient()
jobkey = parse_job_key(os.environ['SHUB_JOBKEY'])
project = client.get_project(jobkey.project_id)
collection_name = input_uri.replace('collections://', '')
self._store = project.collections.get_store(collection_name)
def __init__(self, crawl_url):
""" Initialize and build a connection with Scrapinghub via its api
"""
self._client = ScrapinghubClient(settings.SCRAPINGHUB_APIKEY)
# TODO: need to be revised
self._project_id = self._client.projects.list()[0]
self._project = self._client.get_project(self._project_id)
self._target = crawl_url
self._job = None
self._meta = None
self._state = 'initialized'
def jobRuning00():
# Enter ScrapingHub
# Enter ScrapingHub
apikey = '40f9881d52794d7bb09b9f5ee6d12a3e' # your API key as a string
client = ScrapinghubClient(apikey)
projectID = 410647
project = client.get_project(projectID)
# get spider
spiderID = 'quotes'
spider = project.spiders.get(spiderID)
spider.jobs.run()
def showBooks(request):
global job
job = test
if job is None:
print("adgaegae")
apikey = '88133cc793ab4296b56db8a87eaae1ec'
client = ScrapinghubClient(apikey)
job = client.get_job('223795/1/3')
job = sorted(job.items.list(), key=lambda k: k['score'], reverse=True)
return render(request, 'user_page.html', {'spider_books': job, 'user_fullname':request.user.get_full_name,'myuser_id':request.user.myuser.id})
else:
'''job = sorted(job.items.list(), key=lambda k: k['score'], reverse=True)'''
return render(request, 'user_page.html',{'spider_books': job, 'user_fullname': request.user.get_full_name,'myuser_id': request.user.myuser.id})
def __init__(self, crawler):
settings = crawler.settings
coll_name = settings.get('TARGET_COLLECTION_NAME')
coll_type = settings.get('TARGET_COLLECTION_TYPE', 's')
if not coll_name or not coll_type:
raise NotConfigured('Please set target collection settings.')
current_project_id = os.environ.get('SCRAPY_PROJECT_ID')
project_id = settings.get('HCF_PROJECT_ID', current_project_id)
self.logger = logging.getLogger(__name__)
# if auth is not set explicitly, fallback to SH job-level token
self.client = ScrapinghubClient(settings.get('HCF_AUTH'))
self.project = self.client.get_project(project_id)
self.collection = self.project.collections.get(coll_type, coll_name)
def main():
args = parse_args()
apikey = os.environ.get('SH_APIKEY') or args.apikey
if not apikey:
print('Please set API key')
exit(1)
client = ScrapinghubClient(apikey)
job = client.get_job(args.job)
events = args.func(job)
if args.command == 'errors':
report_errors = create_errors_report(events,
max_urls_for_output=(min(
args.max, 30)))
print(report_errors)
def menu():
client = ScrapinghubClient(config['scrapinghub']['api_key'])
project = client.get_project(config['scrapinghub']['project_id'])
job = project.jobs.list(spider=config['scrapinghub']['spider_name'], state='finished', count=1)[0]
job = client.get_job(job['key'])
menu = {}
menu['aktualnosc'] = job.metadata.get('finished_time')
menu['restauracja'] = {
"nazwa": "CamelPizza",
"logo": "https://www.camelpizza.pl/system/logos/27323/menu_size/1549450693.png",
"url": "http://camelpizza.pl"
}
menu['grupy'] = []
def get_grupa(item):
for grupa in menu['grupy']:
if grupa['nazwa'] == item['grupa']:
return grupa
grupa = { 'nazwa': item['grupa'], 'pozycje': [] }
menu['grupy'].append(grupa)
return grupa
def get_pozycja(item):
grupa = get_grupa(item)
for pozycja in grupa['pozycje']:
if pozycja['nazwa'] == item['pozycja']:
return pozycja
pozycja = { 'nazwa': item['pozycja'], 'opis': item['opis'], 'warianty': [] }
grupa['pozycje'].append(pozycja)
return pozycja
def get_cena(item):
kwota, waluta = item['cena'].replace(u'zł', u' zł').split()
kwota = float(kwota.replace(',', '.'))
waluta = waluta.replace(u'zł', 'PLN')
return { 'kwota': kwota, 'waluta': waluta }
items = job.items.list()
for item in items:
try:
pozycja = get_pozycja(item)
wariant = { 'opis': item['wariant'], 'ceny': [ get_cena(item) ]}
pozycja['warianty'].append(wariant)
except:
print("Invalid item")
return jsonify(menu)
class PttCrawlerJob():
def __init__(self, crawl_url):
""" Initialize and build a connection with Scrapinghub via its api
"""
self._client = ScrapinghubClient(settings.SCRAPINGHUB_APIKEY)
# TODO: need to be revised
self._project_id = self._client.projects.list()[0]
self._project = self._client.get_project(self._project_id)
self._target = crawl_url
self._job = None
self._meta = None
self._state = 'initialized'
def run(self):
""" Run the crawler (spider)
"""
if not self._job:
self._job = self._project.jobs.run(
'ptt', job_args={'test_url': self._target})
return self._job.key
else:
return None
def update_meta(self):
""" Update job's meta data
"""
if self._job:
self._meta = dict(self._job.metadata.iter())
self._state = self._meta['state']
def cancle(self):
""" Cancle a job
"""
self._job.cancel()
@property
def meta(self):
""" Get job's meta data
"""
if self._meta:
return self._meta
else:
return None
@property
def state(self):
""" Get job's current state
"""
return self._state
@property
def item(self):
""" Get scrapped items
"""
if self._state == 'finished':
# items.iter() returns a iterable, but is not a list and does not support indexing
# so it has to be transformed into a list. Each element in the list is a dict.
return list(self._job.items.iter())
else:
return None
def job(self) -> Job:
if not self._job:
job = ScrapinghubClient().get_job(self.key)
if job.metadata.get("state") == "deleted":
raise ValueError(f"{self.key} has 'deleted' state")
self._job = job
return self._job
class ScrapyCloudClient:
def __init__(self):
self.apikey = '' # your API key as a string
self.client = ScrapinghubClient(self.apikey)
self.project_num = 0
self.project = self.client.get_project(self.project_num)
self.neighborhood_spider = self.get_neighborhood_spider()
self.listing_spider = self.get_listing_spider()
self.airdna_spider = self.get_airdna_spider()
def get_neighborhood_spider(self):
return ScrapyCloudNeighborhoodSearchSpider(
self.project.spiders.get('neighborhood_search'))
def get_listing_spider(self):
return ScrapyCloudSpider(self.project.spiders.get('listing'))
def get_airdna_spider(self):
return ScrapyCloudSpider(self.project.spiders.get('airdna'))
def listing_ids(self):
all_ids = self.neighborhood_spider.get_listing_ids()
print(len(all_ids))
id_string = ""
for num, i in enumerate(all_ids):
if num == 0:
id_string = str(i)
else:
id_string = id_string + "," + str(i)
return id_string
def shub_conn():
# don't use default `.get()` property because then it will evaluate
# `settings.SH_API_KEY` anyway and you might not have setup it locally
api_key = os.environ.get('SH_API_KEY') or Settings().get('SH_API_KEY')
# NOTE not really safe when `name` doesn't exist
return ScrapinghubClient(api_key)
class HcfCrawlerPipeline(object):
def __init__(self, crawler):
settings = crawler.settings
coll_name = settings.get('TARGET_COLLECTION_NAME')
coll_type = settings.get('TARGET_COLLECTION_TYPE', 's')
if not coll_name or not coll_type:
raise NotConfigured('Please set target collection settings.')
current_project_id = os.environ.get('SCRAPY_PROJECT_ID')
project_id = settings.get('HCF_PROJECT_ID', current_project_id)
self.logger = logging.getLogger(__name__)
# if auth is not set explicitly, fallback to SH job-level token
self.client = ScrapinghubClient(settings.get('HCF_AUTH'))
self.project = self.client.get_project(project_id)
self.collection = self.project.collections.get(coll_type, coll_name)
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
def process_item(self, item, spider):
item_to_export = dict(item)
if not '_key' in item_to_export:
fp = hashlib.sha1()
fp.update(canonicalize_url(item['url']).encode('utf8'))
item_to_export['_key'] = fp.hexdigest()
self.collection.set(item_to_export)
return item
def start_requests(self):
#self.c.execute('DROP TABLE IF EXISTS iteminfos')
#self.c.execute('CREATE TABLE IF NOT EXISTS iteminfos (item_id, item_main_type, item_mid_type, item_sub_type, item_price, area_id)')
#self.conn.commit()
#temp_conn = sql.connect('dataset/area.db')
#temp_c = temp_conn.cursor()
#temp_c.execute('DELETE FROM areainfos WHERE rowid NOT IN (SELECT min(rowid) FROM areainfos GROUP BY item_id,item_url,area_id)')
#item_list = [row for row in temp_c.execute('SELECT * FROM areainfos ORDER BY area_id')]
#temp_conn.commit()
client = ScrapinghubClient('ec16b94bcf024d0bb502684368658d59')
myproject = client.projects.get('254951')
mystore = myproject.collections.get_store('area_info')
value_num = mystore.count()
#for item in item_list:
for item in range(value_num):
#url = item[1].encode()
log_item = mystore.get(str(item))
area_id = log_item['value']['area_id'][0]
item_ids = log_item['value']['item_id']
item_urls = log_item['value']['item_url']
for i in range(len(item_ids)):
if i % 10 == 0:
sleep(0.8)
#request = scrapy.Request(url=url,callback=self.parse)
#request.meta['item_id'] = item[0]
#request.meta['area_id'] = item[2]
request = scrapy.Request(url=item_urls[i],
callback=self.parse,
errback=self.error_handler)
request.meta['item_id'] = item_ids[i]
request.meta['area_id'] = area_id
yield request
def create_json_schema(source_key: str,
item_numbers: Optional[List[int]] = None) -> dict:
if helpers.is_collection_key(source_key):
store = api.get_collection(source_key)
items_count = store.count()
elif helpers.is_job_key(source_key):
job = ScrapinghubClient().get_job(source_key)
items_count = api.get_items_count(job)
else:
raise ValueError(
f"'{source_key}' is not a valid job or collection key")
if items_count == 0:
raise ValueError(f"'{source_key}' does not have any items")
item_n_err = "{} is a bad item number, choose numbers between 0 and {}"
if item_numbers:
item_numbers.sort()
if item_numbers[-1] >= items_count or item_numbers[0] < 0:
raise ValueError(
item_n_err.format(item_numbers[-1], items_count - 1))
else:
item_numbers = set_item_no(items_count)
samples = []
for n in item_numbers:
items = api.get_items(source_key, start_index=n, count=1, p_bar=None)
samples.append(items[0])
return infer_schema(samples)
def __init__(self, crawler):
self.crawler = crawler
settings = crawler.settings
current_project_id = os.environ.get('SCRAPY_PROJECT_ID')
project_id = settings.get('HCF_PROJECT_ID', current_project_id)
frontier_name = settings.get('HCF_FRONTIER')
frontier_slot = settings.get('HCF_FRONTIER_SLOT')
if not project_id or not frontier_name or not frontier_slot:
raise NotConfigured('Please set HCF settings for the middleware.')
self.batch_size = settings.getint('HCF_BATCH_SIZE', DEFAULT_BATCH_SIZE)
self.logger = logging.getLogger(__name__)
# if auth is not set explicitly, fallback to SH job-level token
self.client = ScrapinghubClient(settings.get('HCF_AUTH'))
self.project = self.client.get_project(project_id)
self.frontier = self.project.frontiers.get(frontier_name)
self.frontier_slot = self.frontier.get(frontier_slot)
def test_projects_list(client):
projects = client.projects.list()
assert client.projects.list() == []
# use user apikey to list test projects
client = ScrapinghubClient(TEST_USER_AUTH, TEST_DASH_ENDPOINT)
projects = client.projects.list()
assert isinstance(projects, list)
assert int(TEST_PROJECT_ID) in projects
def index():
apikey = os.environ.get("APIKEY")
job_id = os.environ.get("JOB_ID")
client = ScrapinghubClient(apikey)
job = client.get_job(job_id)
data = []
for item in job.items.iter():
dict = {
'title': item['title'][0],
'director': item['director'][0],
'summary': item['summary'][0]
}
data.append(dict)
return render_template('index.html', data=data)
def __init__(self):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('pid', help='Taget project id')
parser.add_argument('spider', help='Spider name')
parser.add_argument('frontier', help='Frontier name')
parser.add_argument('prefix', help='Slot prefix')
parser.add_argument('--max-jobs', help='Max number of jobs for the given spider allowed to run in parallel.\
Default is %(default)s.', type=int, default=1)
parser.add_argument('--apikey',
help='API key to use for HCF access. Uses SH_APIKEY environment variable if not given')
parser.add_argument('--spider-args', help='Spider arguments dict in json format', default='{}')
parser.add_argument('--loop-mode', help='If provided, manager will run in loop mode, with a cycle each given\
number of seconds.', type=int, metavar='SECONDS')
self.args = parser.parse_args()
client = ScrapinghubClient(self.args.apikey)
self.project = client.get_project(self.args.pid)
self.hcfpal = HCFPal(client._hsclient.get_project(self.args.pid))
def obtainLatestJobIDofSpider(apikey, project_id, spider_id):
client = ScrapinghubClient(apikey)
myproject = client.projects.get(project_id)
job_keys = [_['key'] for _ in myproject.jobs.iter()]
job_ids = [
int(_.split('/')[2]) if (_.split('/')[0] == myproject.key
and _.split('/')[1] == str(spider_id)) else ''
for _ in job_keys
]
job_ids = [_ if type(_) is int else 0 for _ in job_ids]
return int(sorted(job_ids)[-1])
def has_project_access(project, endpoint, apikey):
"""Check whether an API key has access to a given project. May raise
InvalidAuthException if the API key is invalid (but not if it is valid but
lacks access to the project)"""
client = ScrapinghubClient(apikey, dash_endpoint=endpoint)
try:
return project in client.projects.list()
except ScrapinghubAPIError as e:
if 'Authentication failed' in str(e):
raise InvalidAuthException
else:
raise RemoteErrorException(str(e))
def restore(spider_id, job_id=0, store_name='', *keys):
APIKEY = 'ec16b94bcf024d0bb502684368658d59'
PROJECTID = '254951'
SPIDERID = spider_id
client = ScrapinghubClient(APIKEY)
myproject = client.get_project(PROJECTID)
if job_id == 0:
myjob_id = obtainLatestJobIDofSpider(APIKEY, PROJECTID, SPIDERID)
#job_keys = [_['key'] for _ in myproject.jobs.iter()]
#job_ids = [int(_.split('/')[2]) if (_.split('/')[0]==myproject.key and _.split('/')[1]==SPIDERID) else '' for _ in job_keys]
#myjob_id = sorted(job_ids)[-1]
else:
myjob_id = job_id
myjob = myproject.jobs.get('%s/%s/%d' % (PROJECTID, SPIDERID, myjob_id))
myitem = [_ for _ in myjob.items.iter()]
item_num = len(myitem)
item_container = dict()
for key_i in keys:
item_container[key_i] = [_[key_i] for _ in myitem]
#area_ids = [_['area_id'] for _ in myitem]
#item_ids = [_['item_id'] for _ in myitem]
#item_urls = [_['item_url'] for _ in myitem]
store_names = [_['name'] for _ in myproject.collections.iter()]
if store_name in store_names:
mycollection = myproject.collections.get_store(store_name)
if mycollection.count() > 0:
for _ in mycollection.iter():
mycollection.delete(_['_key'])
for item_i in range(item_num):
area_info_item = dict()
for key_i in keys:
area_info_item[key_i] = item_container[key_i][item_i]
#area_info_item['area_id'] = area_ids[item_i]
#area_info_item['item_id'] = item_ids[item_i]
#area_info_item['item_url'] = item_urls[item_i]
mycollection.set({'_key': str(item_i), 'value': area_info_item})
else:
print "the collection %s you want to access is not exist." % store_name
def schedule_spider(project, endpoint, apikey, spider, arguments=(), settings=(),
priority=DEFAULT_PRIORITY, units=None, tag=(), environment=()):
client = ScrapinghubClient(apikey, dash_endpoint=endpoint)
try:
project = client.get_project(project)
args = dict(x.split('=', 1) for x in arguments)
cmd_args = args.pop('cmd_args', None)
meta = args.pop('meta', None)
job = project.jobs.run(
spider=spider,
meta=json.loads(meta) if meta else {},
cmd_args=cmd_args,
job_args=args,
job_settings=dict(x.split('=', 1) for x in settings),
priority=priority,
units=units,
add_tag=tag,
environment=dict(x.split('=', 1) for x in environment),
)
return job.key
except ScrapinghubAPIError as e:
raise RemoteErrorException(str(e))