def upload_customized_spider(directory=None,
name=None,
col=None,
display_name=None,
command=None,
id=None):
# check if directory exists
if not os.path.exists(directory):
print(f'error: {directory} does not exist')
return
# compress the directory to the zipfile
target_path = os.path.join(CRAWLAB_TMP, name
or os.path.basename(directory)) + '.zip'
zip_dir(directory, target_path)
# upload zip file to server
if id is None:
res = Request.upload('/spiders',
target_path,
data={
'name': name,
'display_name': display_name,
'col': col,
'command': command,
})
if res.get('error'):
print('error: ' + res.get('error'))
return
else:
res = Request.upload(f'/spiders/{id}/upload', target_path)
if res.get('error'):
print('error: ' + res.get('error'))
return
print('uploaded successfully')
def list_nodes():
data = Request.get('/nodes')
if data.get('error'):
print('error: ' + data.get('error'))
items = data.get('data') or []
columns = ['_id', 'name', 'status', 'create_ts', 'update_ts']
Client.list(columns, items)
def list_tasks(number=10):
data = Request.get('/tasks', {'page_num': 1, 'page_size': number})
if data.get('error'):
print('error: ' + data.get('error'))
items = data.get('data') or []
columns = [
'_id', 'status', 'node_name', 'spider_name', 'error',
'result_count', 'create_ts', 'update_ts'
]
Client.list(columns, items)
def list_spiders():
data = Request.get('/spiders', {'page_num': 1, 'page_size': 99999999})
if data.get('error'):
print('error: ' + data.get('error'))
items = (data.get('data').get('list')
or []) if data.get('data') is not None else []
columns = [
'_id', 'name', 'display_name', 'type', 'col', 'create_ts',
'update_ts'
]
Client.list(columns, items)
def update_token():
data = Request.post('/login',
data={
'username': config.data.username,
'password': config.data.password,
})
if data.get('error'):
print('error: ' + data.get('error'))
print('error when logging-in')
return
config.data.token = data.get('data')
config.save()
print('logged-in successfully')
def list_schedules():
data = Request.get('/schedules', {
'page_num': 1,
'page_size': 99999999
})
if data.get('error'):
print('error: ' + data.get('error'))
items = data.get('data') or []
columns = [
'_id', 'name', 'spider_name', 'run_type', 'cron', 'create_ts',
'update_ts'
]
Client.list(columns, items)