Python list_all_blobs Examples

Example #1

File: update_ci_routes.py Project: seanpmorgan/openpilot

def sync_to_ci_public(route):
  print(f"Uploading {route}")
  key_prefix = route.replace('|', '/')
  dongle_id = key_prefix.split('/')[0]

  if next(azureutil.list_all_blobs(SERVICE, "openpilotci", prefix=key_prefix), None) is not None:
    print("Already synced")
    return True

  for (source_account, source_bucket), source_key in zip(SOURCES, SOURCE_KEYS):
    print(f"Trying {source_account}/{source_bucket}")
    cmd = [
      f"{BASEDIR}/external/bin/azcopy",
      "copy",
      "https://{}.blob.core.windows.net/{}/{}/?{}".format(source_account, source_bucket, key_prefix, source_key),
      "https://{}.blob.core.windows.net/{}/{}/?{}".format(_DATA_ACCOUNT_CI, "openpilotci", dongle_id, DEST_KEY),
      "--recursive=true",
      "--overwrite=false",
      "--exclude=*/dcamera.hevc",
    ]

    try:
      result = subprocess.call(cmd, stdout=subprocess.DEVNULL)
      if result == 0:
        print("Success")
        return True
    except subprocess.CalledProcessError:
      print("Failed")

  return False

Example #2

File: update_ci_routes.py Project: michaelhonan/openpilot

def sync_to_ci_public(route):
    key_prefix = route.replace('|', '/')
    dongle_id = key_prefix.split('/')[0]

    if next(
            azureutil.list_all_blobs(SERVICE, "openpilotci",
                                     prefix=key_prefix), None) is not None:
        return True

    print(f"Uploading {route}")
    for (source_account,
         source_bucket), source_key in zip(SOURCES, SOURCE_KEYS):
        print(f"Trying {source_account}/{source_bucket}")
        cmd = [
            "azcopy",
            "copy",
            f"https://{source_account}.blob.core.windows.net/{source_bucket}/{key_prefix}?{source_key}",
            f"https://{_DATA_ACCOUNT_CI}.blob.core.windows.net/openpilotci/{dongle_id}?{DEST_KEY}",
            "--recursive=true",
            "--overwrite=false",
            "--exclude-pattern=*/dcamera.hevc",
        ]

        try:
            result = subprocess.call(cmd, stdout=subprocess.DEVNULL)
            if result == 0:
                print("Success")
                return True
        except subprocess.CalledProcessError:
            print("Failed")

    return False

Example #3

def sync_to_ci_public(service, route):
    key_prefix = route.replace('|', '/')

    if next(
            azureutil.list_all_blobs(service, "openpilotci",
                                     prefix=key_prefix), None) is not None:
        return

    print("uploading", route)

    tmpdir = tempfile.mkdtemp()
    try:
        print(
            f"download_dir_tpe({_DATA_ACCOUNT_PRODUCTION}, {_DATA_BUCKET_PRODUCTION}, {key_prefix}, {tmpdir})"
        )

        # production -> openpilotci
        download_dir_tpe(_DATA_ACCOUNT_PRODUCTION, _DATA_BUCKET_PRODUCTION,
                         tmpdir, key_prefix)

        # commadataci -> openpilotci
        #download_dir_tpe(_DATA_ACCOUNT_CI, _DATA_BUCKET_CI, tmpdir, key_prefix)

        upload_dir_serial(_DATA_ACCOUNT_CI, "openpilotci", tmpdir, key_prefix)
    finally:
        shutil.rmtree(tmpdir)