def set_config(cls, config=None):
config_file_path = cls.get_config_file_path()
if os.path.isfile(config_file_path):
logger.debug("{} file already present at {}".format(
cls.CONFIG_FILE_NAME, config_file_path))
return
logger.debug("Setting default {} in the file {}".format(
cls.CONFIG_FILE_NAME, config_file_path))
with open(config_file_path, "w") as config_file:
config_file.write(config or DEFAULT_IGNORE_LIST)
def download(self, url, filename, relative=False, headers=None, timeout=5):
"""
Download the file from the given url at the current path
"""
request_url = self.base_url + url if relative else url
logger.debug("Downloading file from url: {}".format(request_url))
request_headers = self._get_headers(headers=headers)
try:
response = requests.get(request_url,
headers=request_headers,
timeout=timeout,
stream=True)
self.check_response_status(response, request_url)
with open(filename, 'wb') as f:
# chunk mode response doesn't have content-length so we are
# using a custom header here
content_length = response.headers.get(
'x-polyaxon-content-length')
if not content_length:
content_length = response.headers.get('content-length')
if content_length:
for chunk in progress.bar(
response.iter_content(chunk_size=1024),
expected_size=(int(content_length) / 1024) + 1):
if chunk:
f.write(chunk)
else:
for chunk in response.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
return filename
except requests.exceptions.ConnectionError as exception:
logger.debug("Exception: {}".format(exception))
sys.exit(
"Cannot connect to the Polyaxon server. Check your internet connection."
)
def upload(self):
try:
upload_files, total_file_size = get_files_in_current_directory(
file_type='code')
except OSError:
sys.exit("Directory contains too many files to upload. "
"If you have data files in the current directory, "
"please upload them separately using `polyaxon data` "
"command and remove them from here.\n"
"See http://docs.polyaxon.com/faqs/job/ "
"for more details on how to fix this.")
logger.info("Creating project run. Total upload size: %s",
total_file_size)
logger.debug("Creating module. Uploading: %s files", len(upload_files))
logger.info("Syncing code ...")
# Add request data
multipart_encoder = MultipartEncoder(fields=upload_files)
# Attach progress bar
progress_callback, bar = create_progress_callback(multipart_encoder)
multipart_encoder_monitor = MultipartEncoderMonitor(
multipart_encoder, progress_callback)
try:
response = self.request(
"POST",
self._get_url(),
data=multipart_encoder_monitor,
headers={"Content-Type": multipart_encoder.content_type},
timeout=3600)
finally:
# always make sure we clear the console
bar.done()
return response.json().get("id")
def request(self,
method,
url,
params=None,
data=None,
files=None,
json=None,
timeout=5,
headers=None):
"""Send a request with the given data as json to the given URL.
Args:
method: HTTP method
url: The un-formatted url to send the request to.
params: Dictionary of values to format url.
data:
files:
json:
timeout:
headers: extra headers to add to the request.
Returns:
Request response if successful, Exception otherwise.
Raises:
PolyaxonHTTPError or one of its subclasses.
"""
request_url = self._get_url(url)
logger.debug(
"Starting request to url: {} with params: {}, data: {}".format(
request_url, params, data))
request_headers = self._get_headers(headers=headers)
try:
response = requests.request(method,
request_url,
params=params,
data=data,
json=json,
headers=request_headers,
files=files,
timeout=timeout)
except requests.exceptions.ConnectionError as exception:
logger.debug("Exception: %s", exception, exc_info=True)
sys.exit(
"Cannot connect to the Polyaxon server. Check your internet connection."
)
logger.debug("Response Content: %s, Headers: %s" %
(response.content, response.headers))
self.check_response_status(response, request_url)
return response
def get_unignored_file_paths(cls, ignore_list=None, white_list=None):
config = cls.get_config()
ignore_list = ignore_list or config[0]
white_list = white_list or config[1]
unignored_files = []
for root, dirs, files in os.walk("."):
logger.debug("Root:%s, Dirs:%s", root, dirs)
if cls.ignore_path(unix_style_path(root), ignore_list, white_list):
dirs[:] = []
logger.debug("Ignoring directory : %s", root)
continue
for file_name in files:
file_path = unix_style_path(os.path.join(root, file_name))
if cls.ignore_path(file_path, ignore_list, white_list):
logger.debug("Ignoring file : %s", file_name)
continue
unignored_files.append(os.path.join(root, file_name))
return unignored_files
def get_cli_version(self):
response = self.get(self._get_url())
data_dict = response.json()
logger.debug("CLI Version info :{}".format(data_dict))
return CliVersionConfig.from_dict(data_dict)
def set_config(cls, config):
config_file_path = cls.get_config_file_path()
logger.debug("Setting {} in the file {}".format(config.to_dict(), cls.CONFIG_FILE_NAME))
with open(config_file_path, "w") as config_file:
config_file.write(json.dumps(config.to_dict()))