def make_deploy_request(url, data, files, auth, verbose, keep_log): last_logs = deque(maxlen=LAST_N_LOGS) try: rsp = requests.post(url=url, auth=auth, data=data, files=files, stream=True, timeout=300) rsp.raise_for_status() write_and_echo_logs(keep_log, last_logs, rsp, verbose) return True except requests.HTTPError as exc: rsp = exc.response if rsp.status_code == 403: raise InvalidAuthException try: error = rsp.json()['message'] if 'Traceback' in error: error = ('\n---------- REMOTE TRACEBACK ----------\n' + error + '\n---------- END OF REMOTE TRACEBACK ----------') except (ValueError, TypeError, KeyError): error = rsp.text msg = "Deploy failed ({}):\n{}".format(rsp.status_code, error) raise RemoteErrorException(msg) except requests.RequestException as exc: raise RemoteErrorException("Deploy failed: {}".format(exc))
def schedule_spider(project, endpoint, apikey, spider, arguments=(), settings=(), priority=DEFAULT_PRIORITY, units=None, tag=(), environment=()): client = ScrapinghubClient(apikey, dash_endpoint=endpoint) try: project = client.get_project(project) args = dict(x.split('=', 1) for x in arguments) cmd_args = args.pop('cmd_args', None) meta = args.pop('meta', None) job = project.jobs.run( spider=spider, meta=json.loads(meta) if meta else {}, cmd_args=cmd_args, job_args=args, job_settings=dict(x.split('=', 1) for x in settings), priority=priority, units=units, add_tag=tag, environment=dict(x.split('=', 1) for x in environment), ) return job.key except ScrapinghubAPIError as e: raise RemoteErrorException(str(e))
def _has_project_access(project, endpoint, apikey): conn = Connection(apikey, url=endpoint) try: return project in conn.project_ids() except APIError as e: if 'Authentication failed' in e.message: raise InvalidAuthException else: raise RemoteErrorException(e.message)
def schedule_spider(project, endpoint, apikey, spider, arguments=(), settings=()): conn = Connection(apikey, url=endpoint) try: return conn[project].schedule( spider, job_settings=json.dumps(dict(x.split('=', 1) for x in settings)), **dict(x.split('=', 1) for x in arguments) ) except APIError as e: raise RemoteErrorException(str(e))
def get_available_projects(): try: resp = requests.get(AVAILABLE_PROJECTS_URL) resp.raise_for_status() except (requests.HTTPError, requests.ConnectionError) as e: raise RemoteErrorException( "There was an error while getting the list of available projects " "from GitHub: %s.\n\nPlease check your connection or go to\n %s\n" "to browse the custom image examples manually." "" % (e, "https://github.com/%s" % EXAMPLE_REPO)) return yaml.safe_load(resp.text)
def has_project_access(project, endpoint, apikey): """Check whether an API key has access to a given project. May raise InvalidAuthException if the API key is invalid (but not if it is valid but lacks access to the project)""" client = ScrapinghubClient(apikey, dash_endpoint=endpoint) try: return project in client.projects.list() except ScrapinghubAPIError as e: if 'Authentication failed' in str(e): raise InvalidAuthException else: raise RemoteErrorException(str(e))
def has_project_access(project, endpoint, apikey): """Check whether an API key has access to a given project. May raise InvalidAuthException if the API key is invalid (but not if it is valid but lacks access to the project)""" conn = Connection(apikey, url=endpoint) try: return project in conn.project_ids() except APIError as e: if 'Authentication failed' in str(e): raise InvalidAuthException else: raise RemoteErrorException(str(e))
def schedule_spider(project, endpoint, apikey, spider, arguments=(), settings=(), priority=DEFAULT_PRIORITY, units=None, tag=()): conn = Connection(apikey, url=endpoint) try: kw = dict(x.split('=', 1) for x in arguments) if units is not None: kw['units'] = units return conn[project].schedule( spider, job_settings=json.dumps(dict(x.split('=', 1) for x in settings)), priority=priority, add_tag=tag, **kw ) except APIError as e: raise RemoteErrorException(str(e))
def write_and_echo_logs(keep_log, last_logs, rsp, verbose): """It will write logs to temporal file and echo if verbose is True.""" with NamedTemporaryFile(prefix='shub_deploy_', suffix='.log', delete=(not keep_log)) as log_file: for line in rsp.iter_lines(): if verbose: click.echo(line) last_logs.append(line) log_file.write(line + b'\n') deployed = _is_deploy_successful(last_logs) echo_short_log_if_deployed(deployed, last_logs, log_file, verbose) if not log_file.delete: click.echo("Deploy log location: %s" % log_file.name) if not deployed: try: last_log = last_logs[-1] except IndexError: last_log = "(No log messages)" raise RemoteErrorException("Deploy failed: {}".format(last_log))
def handle_event(self, event): if 'error' in event: tqdm.write("Error {}: {}".format(event['error'], event['errorDetail'])) raise RemoteErrorException("Docker operation failed")
def _assert_response_is_valid(rsp): if rsp.status_code == 403: raise InvalidAuthException elif rsp.status_code != 200: msg = 'Eggs could not be fetched. Status: %d' % rsp.status_code raise RemoteErrorException(msg)