def make_deploy_request(url, data, files, auth, verbose, keep_log):
last_logs = deque(maxlen=LAST_N_LOGS)
try:
rsp = requests.post(url=url,
auth=auth,
data=data,
files=files,
stream=True,
timeout=300)
rsp.raise_for_status()
write_and_echo_logs(keep_log, last_logs, rsp, verbose)
return True
except requests.HTTPError as exc:
rsp = exc.response
if rsp.status_code == 403:
raise InvalidAuthException
try:
error = rsp.json()['message']
if 'Traceback' in error:
error = ('\n---------- REMOTE TRACEBACK ----------\n' + error +
'\n---------- END OF REMOTE TRACEBACK ----------')
except (ValueError, TypeError, KeyError):
error = rsp.text
msg = "Deploy failed ({}):\n{}".format(rsp.status_code, error)
raise RemoteErrorException(msg)
except requests.RequestException as exc:
raise RemoteErrorException("Deploy failed: {}".format(exc))
def schedule_spider(project,
endpoint,
apikey,
spider,
arguments=(),
settings=(),
priority=DEFAULT_PRIORITY,
units=None,
tag=(),
environment=()):
client = ScrapinghubClient(apikey, dash_endpoint=endpoint)
try:
project = client.get_project(project)
args = dict(x.split('=', 1) for x in arguments)
cmd_args = args.pop('cmd_args', None)
meta = args.pop('meta', None)
job = project.jobs.run(
spider=spider,
meta=json.loads(meta) if meta else {},
cmd_args=cmd_args,
job_args=args,
job_settings=dict(x.split('=', 1) for x in settings),
priority=priority,
units=units,
add_tag=tag,
environment=dict(x.split('=', 1) for x in environment),
)
return job.key
except ScrapinghubAPIError as e:
raise RemoteErrorException(str(e))
def _has_project_access(project, endpoint, apikey):
conn = Connection(apikey, url=endpoint)
try:
return project in conn.project_ids()
except APIError as e:
if 'Authentication failed' in e.message:
raise InvalidAuthException
else:
raise RemoteErrorException(e.message)
def schedule_spider(project, endpoint, apikey, spider, arguments=(),
settings=()):
conn = Connection(apikey, url=endpoint)
try:
return conn[project].schedule(
spider,
job_settings=json.dumps(dict(x.split('=', 1) for x in settings)),
**dict(x.split('=', 1) for x in arguments)
)
except APIError as e:
raise RemoteErrorException(str(e))
def get_available_projects():
try:
resp = requests.get(AVAILABLE_PROJECTS_URL)
resp.raise_for_status()
except (requests.HTTPError, requests.ConnectionError) as e:
raise RemoteErrorException(
"There was an error while getting the list of available projects "
"from GitHub: %s.\n\nPlease check your connection or go to\n %s\n"
"to browse the custom image examples manually."
"" % (e, "https://github.com/%s" % EXAMPLE_REPO))
return yaml.safe_load(resp.text)
def has_project_access(project, endpoint, apikey):
"""Check whether an API key has access to a given project. May raise
InvalidAuthException if the API key is invalid (but not if it is valid but
lacks access to the project)"""
client = ScrapinghubClient(apikey, dash_endpoint=endpoint)
try:
return project in client.projects.list()
except ScrapinghubAPIError as e:
if 'Authentication failed' in str(e):
raise InvalidAuthException
else:
raise RemoteErrorException(str(e))
def has_project_access(project, endpoint, apikey):
"""Check whether an API key has access to a given project. May raise
InvalidAuthException if the API key is invalid (but not if it is valid but
lacks access to the project)"""
conn = Connection(apikey, url=endpoint)
try:
return project in conn.project_ids()
except APIError as e:
if 'Authentication failed' in str(e):
raise InvalidAuthException
else:
raise RemoteErrorException(str(e))
def schedule_spider(project, endpoint, apikey, spider, arguments=(), settings=(),
priority=DEFAULT_PRIORITY, units=None, tag=()):
conn = Connection(apikey, url=endpoint)
try:
kw = dict(x.split('=', 1) for x in arguments)
if units is not None:
kw['units'] = units
return conn[project].schedule(
spider,
job_settings=json.dumps(dict(x.split('=', 1) for x in settings)),
priority=priority,
add_tag=tag,
**kw
)
except APIError as e:
raise RemoteErrorException(str(e))
def write_and_echo_logs(keep_log, last_logs, rsp, verbose):
"""It will write logs to temporal file and echo if verbose is True."""
with NamedTemporaryFile(prefix='shub_deploy_', suffix='.log',
delete=(not keep_log)) as log_file:
for line in rsp.iter_lines():
if verbose:
click.echo(line)
last_logs.append(line)
log_file.write(line + b'\n')
deployed = _is_deploy_successful(last_logs)
echo_short_log_if_deployed(deployed, last_logs, log_file, verbose)
if not log_file.delete:
click.echo("Deploy log location: %s" % log_file.name)
if not deployed:
try:
last_log = last_logs[-1]
except IndexError:
last_log = "(No log messages)"
raise RemoteErrorException("Deploy failed: {}".format(last_log))
def handle_event(self, event):
if 'error' in event:
tqdm.write("Error {}: {}".format(event['error'],
event['errorDetail']))
raise RemoteErrorException("Docker operation failed")
def _assert_response_is_valid(rsp):
if rsp.status_code == 403:
raise InvalidAuthException
elif rsp.status_code != 200:
msg = 'Eggs could not be fetched. Status: %d' % rsp.status_code
raise RemoteErrorException(msg)
