def i_delete_one_of_the_chunks(step): manifest = release_local_data() remote_dir, manifest_name = os.path.split(world.manifest_url) chunk, md5sum, size = choice(choice(manifest["files"])["chunks"]) chunk_url = os.path.join(remote_dir, chunk) world.deleted_chunk = chunk_url LOG.debug("Lettuce deleting %s" % chunk_url) world.driver.delete(chunk_url)
def i_replace_the_manifest_with_old_repr(step): manifest = release_local_data() manifest_ini_path = os.path.join(world.basedir, "manifest.ini") with open(manifest_ini_path, 'w') as fd: convert_manifest(manifest).write(fd) world.driver.delete(world.manifest_url) world.manifest_url = world.driver.put(manifest_ini_path, os.path.join(os.path.dirname(world.manifest_url), '')) LOG.debug("NEW %s" % world.manifest_url)
def i_upload_stream_with_gzipping(context, gzip): gzip == 'True' src = [cloudfs.NamedStream(stream.src, os.path.basename(stream.src.name)) for stream in context.sources] for s in src: LOG.debug(s.name) time.sleep(1) if len(src) == 1: src = src[0] context.tr = largetransfer.Upload(src, context.remote_dir, gzip=gzip, chunk_size=5) context.tr.apply_async() try: context.tr.join() except: context.error = sys.exc_info() context.manifest = context.tr.manifest context.gzip = True
def convert_manifest(json_manifest): assert len(json_manifest["files"]) == 1 assert json_manifest["files"][0]["compressor"] == "gzip" parser = ConfigParser() parser.add_section("snapshot") parser.add_section("chunks") parser.set("snapshot", "description", json_manifest["description"]) parser.set("snapshot", "created_at", json_manifest["created_at"]) parser.set("snapshot", "pack_method", json_manifest["files"][0]["compressor"]) for chunk, md5sum, size in reversed(json_manifest["files"][0]["chunks"]): parser.set("chunks", chunk, md5sum) LOG.debug("CONVERT: %s", parser.items("chunks")) return parser
def make_file(base_dir, name=None, size=None): name = name or uuid.uuid4().hex file_path = os.path.join(base_dir, name) if size is None: size = random.randint(2, 20) if not os.path.exists(base_dir): os.mkdir(base_dir) subprocess.call([ "dd", "if=/dev/urandom", "of=%s" % file_path, "bs=1M", "count=%s" % size ], stdout=open('/dev/null', 'w'), stderr=subprocess.STDOUT, close_fds=True) LOG.debug('Make file %s, %s' % (file_path, size)) assert os.path.exists(file_path), file_path return file_path, size
import os import shutil import tempfile import mock import logging import boto import swiftclient from scalarizr.storage2 import cloudfs from scalarizr.storage2.cloudfs import s3, gcs, swift, local from scalarizr.storage2.cloudfs import LOG LOG.setLevel(logging.DEBUG) LOG.addHandler(logging.FileHandler("transfer_test.log", 'w')) import httplib2 from apiclient.discovery import build from oauth2client.client import SignedJwtAssertionCredentials def get_s3_conn(): conn = boto.connect_s3( host='s3.amazonaws.com', aws_access_key_id=os.environ['EC2_ACCESS_KEY'], aws_secret_access_key=os.environ['EC2_SECRET_KEY']) return conn
"http://folsom.enter.it:5000/v2.0", os.environ["ENTER_IT_USERNAME"], os.environ["ENTER_IT_API_KEY"], auth_version="2") @after.each_feature @this_feature_only def teardown_feature(feat): for args in _RESTORE: setattr(*args) # # Logging # LOG.setLevel(logging.DEBUG) LOG.addHandler(logging.FileHandler("transfer_test.log", 'w')) # # # STORAGES = { "s3": { "url": "s3://vova-new/vova_test", "driver": s3.S3FileSystem, }, "gcs": { "url": "gcs://vova-test",