Beispiel #1
0
    def test_get_bucket(self):
        config.Bucket(
            id='master.tryserver.chromium.linux',
            project_id='chromium',
            revision='deadbeef',
            config_content=MASTER_TRYSERVER_CHROMIUM_LINUX_CONFIG_TEXT,
            config_content_binary=text_to_binary(
                MASTER_TRYSERVER_CHROMIUM_LINUX_CONFIG_TEXT),
        ).put()
        project, cfg = config.get_bucket('master.tryserver.chromium.linux')
        self.assertEqual(project, 'chromium')
        self.assertEqual(
            cfg,
            project_config_pb2.Bucket(
                name='master.tryserver.chromium.linux',
                acls=[
                    project_config_pb2.Acl(role=project_config_pb2.Acl.READER,
                                           group='all'),
                    project_config_pb2.Acl(
                        role=project_config_pb2.Acl.SCHEDULER,
                        group='tryjob-access'),
                ]),
        )

        self.assertIsNone(config.get_bucket('non.existing')[0])
Beispiel #2
0
    def test_get_bucket(self):
        config.put_bucket('chromium', 'deadbeef', LUCI_CHROMIUM_TRY)
        rev, cfg = config.get_bucket('chromium/try')
        self.assertEqual(rev, 'deadbeef')
        self.assertEqual(cfg, short_bucket_cfg(LUCI_CHROMIUM_TRY))

        self.assertIsNone(config.get_bucket('chromium/nonexistent')[0])
Beispiel #3
0
def _reject_swarming_bucket(bucket_id):
    config.validate_bucket_id(bucket_id)
    _, cfg = config.get_bucket(bucket_id)
    assert cfg, 'permission check should have failed'
    if config.is_swarming_config(cfg):
        raise errors.InvalidInputError(
            'Invalid operation on a Swarming bucket')
Beispiel #4
0
 def get_bucket(self, request):
   """Returns bucket information."""
   bucket_id = convert_bucket(request.bucket)  # checks access
   project_id, _ = config.parse_bucket_id(bucket_id)
   rev, bucket_cfg = config.get_bucket(bucket_id)
   assert bucket_cfg  # access check would have failed.
   return BucketMessage(
       name=request.bucket,
       project_id=project_id,
       config_file_content=protobuf.text_format.MessageToString(bucket_cfg),
       config_file_rev=rev,
       config_file_url=config.get_buildbucket_cfg_url(project_id),
   )
Beispiel #5
0
  def test_cron_update_buckets_with_broken_configs(self, get_project_configs):
    config.put_bucket('chromium', 'deadbeef', MASTER_TRYSERVER_CHROMIUM_LINUX)

    get_project_configs.return_value = {
        'chromium': (
            'new!', None, config_component.ConfigFormatError('broken!')
        ),
    }

    config.cron_update_buckets()

    # We must not delete buckets defined in a project that currently have a
    # broken config.
    bucket_id = 'chromium/' + MASTER_TRYSERVER_CHROMIUM_LINUX.name
    _, actual = config.get_bucket(bucket_id)
    self.assertEqual(actual, MASTER_TRYSERVER_CHROMIUM_LINUX)
Beispiel #6
0
    def set_next_build_number(self, request):
        """Sets the build number that will be used for the next build."""
        bucket_id = api.convert_bucket(request.bucket)
        if not user.can_set_next_number_async(bucket_id).get_result():
            raise endpoints.ForbiddenException('access denied')
        _, bucket = config.get_bucket(bucket_id)

        if not any(b.name == request.builder
                   for b in bucket.swarming.builders):
            raise endpoints.BadRequestException(
                'builder "%s" not found in bucket "%s"' %
                (request.builder, bucket_id))

        project, bucket = config.parse_bucket_id(bucket_id)
        builder_id = build_pb2.BuilderID(project=project,
                                         bucket=bucket,
                                         builder=request.builder)
        seq_name = sequence.builder_seq_name(builder_id)
        try:
            sequence.set_next(seq_name, request.next_number)
        except ValueError as ex:
            raise endpoints.BadRequestException(str(ex))
        return message_types.VoidMessage()
import petl as etl
import cx_Oracle
import boto3
from passyunk.parser import PassyunkParser
from config import get_dsn, get_bucket

month = '2018_09'
parser = PassyunkParser()

# Input locations
loc = r'C:/Projects/etl/data/usps/'
csbyst = '/pa.txt'
zip4 = '/pa'

# Output params
s3_bucket = get_bucket()
dsn = get_dsn('ais')
connection = cx_Oracle.Connection(dsn)
zip4_write_table_name = 'USPS_ZIP4S'
cityzip_write_table_name = 'USPS_CITYZIP'
alias_write_table_name = 'USPS_ALIAS'
address_standardization_report_table_name = 'USPS_ZIP4_ADDRESS_CHECK'
alias_outfile_path = alias_write_table_name.lower() + '.csv'
cityzip_outfile_path = cityzip_write_table_name.lower() + '.csv'
zip4_outfile_path = zip4_write_table_name.lower() + '.csv'
temp_zip4_outfile_path = 'T_' + zip4_outfile_path
#####################################
# Meta:

zip3s = ['190', '191', '192']
zips = ['19019',
Beispiel #8
0
import petl as etl
import cx_Oracle
import boto3
from passyunk.parser import PassyunkParser
from config import get_dsn, get_bucket

month = '2020_03'
parser = PassyunkParser()

# Input locations
loc = r'C:/Projects/etl/data/usps/'
csbyst = '/pa.txt'
zip4 = '/pa'

# Output params
s3_bucket = get_bucket()
dsn = get_dsn('ais')
connection = cx_Oracle.Connection(dsn)
zip4_write_table_name = 'USPS_ZIP4S'
cityzip_write_table_name = 'USPS_CITYZIP'
alias_write_table_name = 'USPS_ALIAS'
address_standardization_report_table_name = 'USPS_ZIP4_ADDRESS_CHECK'
alias_outfile_path = alias_write_table_name.lower() + '.csv'
cityzip_outfile_path = cityzip_write_table_name.lower() + '.csv'
zip4_outfile_path = zip4_write_table_name.lower() + '.csv'
temp_zip4_outfile_path = 'T_' + zip4_outfile_path
#####################################
# Meta:

zip3s = ['190', '191', '192']
zips = [