def _hashTestFile(self, url):
import boto
from toil.jobStores.googleJobStore import GoogleJobStore
projectID, uri = GoogleJobStore._getResources(urlparse.urlparse(url))
uri = boto.storage_uri(uri)
contents = uri.get_contents_as_string(headers=self.headers)
return hashlib.md5(contents).hexdigest()
def _hashTestFile(self, url):
import boto
from toil.jobStores.googleJobStore import GoogleJobStore
projectID, uri = GoogleJobStore._getResources(urlparse.urlparse(url))
uri = boto.storage_uri(uri)
contents = uri.get_contents_as_string(headers=self.headers)
return hashlib.md5(contents).hexdigest()
def _hashUrl(url):
import boto
from toil.jobStores.googleJobStore import GoogleJobStore
projectID, uri = GoogleJobStore._getResources(urlparse.urlparse(url))
return hashlib.md5(
boto.storage_uri(uri).get_contents_as_string(
headers=GoogleJobStoreTest.headers)).hexdigest()
def _cleanUpExternalStore(url):
import boto
from toil.jobStores.googleJobStore import GoogleJobStore
projectID, uri = GoogleJobStore._getResources(urlparse.urlparse(url))
uri = boto.storage_uri(uri)
headers = {"x-goog-project-id": projectID}
bucket = uri.get_bucket(headers=headers, validate=True)
if bucket is not None:
while True:
for key in bucket.list():
try:
key.delete()
except boto.exception.GSResponseError as e:
if e.status == 404:
pass
else:
raise e
try:
uri.delete_bucket()
except boto.exception.GSResponseError as e:
if e.status == 404:
break
else:
continue
def _cleanUpExternalStore(url):
import boto
from toil.jobStores.googleJobStore import GoogleJobStore
projectID, uri = GoogleJobStore._getResources(urlparse.urlparse(url))
uri = boto.storage_uri(uri)
headers = {"x-goog-project-id": projectID}
bucket = uri.get_bucket(headers=headers, validate=True)
if bucket is not None:
while True:
for key in bucket.list():
try:
key.delete()
except boto.exception.GSResponseError as e:
if e.status == 404:
pass
else:
raise e
try:
uri.delete_bucket()
except boto.exception.GSResponseError as e:
if e.status == 404:
break
else:
continue
def _hashUrl(url):
import boto
from toil.jobStores.googleJobStore import GoogleJobStore
projectID, uri = GoogleJobStore._getResources(urlparse.urlparse(url))
return hashlib.md5(boto.storage_uri(uri).get_contents_as_string(headers=GoogleJobStoreTest.headers)).hexdigest()
