def run(config, filename, debug, limit, offset, resource):
sys.stdout.write(filename + ' ... ')
sys.stdout.flush()
configparser = metl.configparser.ConfigParser(
metl.config.Config(config),
debug=debug,
limit=limit,
offset=offset,
source_resource=resource)
metl.manager.Manager(configparser.getTarget()).run()
sys.stdout.write('OK\n')
sys.stdout.flush()
def write(filepath, records):
source = metl.source.staticsource.StaticSource(
metl.fieldset.FieldSet(fields=[
metl.field.Field(
'key', metl.fieldtype.stringfieldtype.StringFieldType())
],
fieldmap=metl.fieldmap.FieldMap({'key':
0})))
source.setResource(sourceRecords=[[r] for r in records])
configparser = metl.configparser.ConfigParser(
metl.config.Config(filepath), init_on_start=False)
configparser.readers = [source]
configparser.loadTarget()
metl.manager.Manager(configparser.getTarget()).run()
def run( config, filename, debug, limit, offset, resource ):
sys.stdout.write( filename + ' ... ' )
sys.stdout.flush()
configparser = metl.configparser.ConfigParser(
metl.config.Config( config ),
debug = debug,
limit = limit,
offset = offset,
source_resource = resource
)
metl.manager.Manager(
configparser.getTarget()
).run()
sys.stdout.write( 'OK\n' )
sys.stdout.flush()
def write( filepath, records ):
source = metl.source.staticsource.StaticSource(
metl.fieldset.FieldSet(
fields = [
metl.field.Field( 'key', metl.fieldtype.stringfieldtype.StringFieldType() )
],
fieldmap = metl.fieldmap.FieldMap({
'key': 0
})
)
)
source.setResource( sourceRecords = [ [r] for r in records ] )
configparser = metl.configparser.ConfigParser( metl.config.Config( filepath ), init_on_start = False )
configparser.readers = [ source ]
configparser.loadTarget()
metl.manager.Manager(
configparser.getTarget()
).run()
def metl_aggregate( argv = sys.argv ):
parser = optparse.OptionParser(
usage = 'Usage: %prog [options] CONFIG.YML FIELD'
)
parser.add_option(
"-p",
"--path",
dest = "path",
default = None,
help = "Conveyance of a folder, which is added to the PATH variable in order that the link in the YAML configuration could be run on an outside python file."
)
parser.add_option(
"-d",
"--debug",
action = 'store_true',
default = False,
help = "Debug mode, writes everything out as stdout."
)
parser.add_option(
"-l",
"--limit",
help = "Determines the number of elements for processing. An excellent option to test large files within small records until everything works as we would like to."
)
parser.add_option(
"-o",
"--offset",
default = 0,
help = 'Determines from which element should the processing start.'
)
parser.add_option(
"-s",
"--source",
help = "If the configuration does not contain the route of the resource, it could be given here as well."
)
(options, args) = parser.parse_args( argv[1:] )
if len( args ) != 2:
parser.print_help()
sys.exit()
if options.path is not None:
absdirectory = os.path.abspath( options.path )
sys.path.append( absdirectory )
configparser = metl.configparser.ConfigParser(
metl.config.Config( args[0] ),
debug = options.debug,
limit = options.limit,
offset = options.offset,
source_resource = options.source
)
target = metl.target.statictarget.StaticTarget( configparser.getTarget().getReader() )
target.setResource( silence = True )
target.initialize()
target.write()
target.finalize()
values = set([])
for record in target.getResults():
values.add( record.getField( args[1] ).getValue() )
for value in values:
print value
def main( argv = sys.argv ):
parser = optparse.OptionParser(
usage = 'Usage: %prog [options] CONFIG.YML'
)
parser.add_option(
"-t",
"--targetMigration",
dest = "target_migration_file",
default = None,
help = "During running, it prepares a migration file from the state of the present data."
)
parser.add_option(
"-m",
"--migration",
dest = "migration_file",
default = None,
help = 'Conveyance of previous migration file that was part of the previously run version.'
)
parser.add_option(
"-p",
"--path",
dest = "path",
default = None,
help = "Conveyance of a folder, which is added to the PATH variable in order that the link in the YAML configuration could be run on an outside python file."
)
parser.add_option(
"-d",
"--debug",
action = 'store_true',
default = False,
help = "Debug mode, writes everything out as stdout."
)
parser.add_option(
"-l",
"--limit",
help = "Determines the number of elements for processing."
)
parser.add_option(
"-o",
"--offset",
default = 0,
help = 'Determines from which element should the processing start.'
)
parser.add_option(
"-s",
"--source",
help = "If the configuration does not contain the route of the resource, it could be given here as well."
)
(options, args) = parser.parse_args( argv[1:] )
if len( args ) != 1:
parser.print_help()
sys.exit()
if options.path is not None:
absdirectory = os.path.abspath( options.path )
sys.path.append( absdirectory )
configparser = metl.configparser.ConfigParser(
metl.config.Config( args[0] ),
debug = options.debug,
limit = options.limit,
offset = options.offset,
source_resource = options.source
)
metl.manager.Manager(
configparser.getTarget(),
migration_resource = options.migration_file,
target_migration_resource = options.target_migration_file
).run()
def metl_aggregate(argv=sys.argv):
parser = optparse.OptionParser(
usage='Usage: %prog [options] CONFIG.YML FIELD')
parser.add_option(
"-p",
"--path",
dest="path",
default=None,
help=
"Conveyance of a folder, which is added to the PATH variable in order that the link in the YAML configuration could be run on an outside python file."
)
parser.add_option("-d",
"--debug",
action='store_true',
default=False,
help="Debug mode, writes everything out as stdout.")
parser.add_option(
"-l",
"--limit",
help=
"Determines the number of elements for processing. An excellent option to test large files within small records until everything works as we would like to."
)
parser.add_option(
"-o",
"--offset",
default=0,
help='Determines from which element should the processing start.')
parser.add_option(
"-s",
"--source",
help=
"If the configuration does not contain the route of the resource, it could be given here as well."
)
(options, args) = parser.parse_args(argv[1:])
if len(args) != 2:
parser.print_help()
sys.exit()
if options.path is not None:
absdirectory = os.path.abspath(options.path)
sys.path.append(absdirectory)
configparser = metl.configparser.ConfigParser(
metl.config.Config(args[0]),
debug=options.debug,
limit=options.limit,
offset=options.offset,
source_resource=options.source)
target = metl.target.statictarget.StaticTarget(
configparser.getTarget().getReader())
target.setResource(silence=True)
target.initialize()
target.write()
target.finalize()
values = set([])
for record in target.getResults():
values.add(record.getField(args[1]).getValue())
for value in values:
print value
def main(argv=sys.argv):
parser = optparse.OptionParser(usage='Usage: %prog [options] CONFIG.YML')
parser.add_option(
"-t",
"--targetMigration",
dest="target_migration_file",
default=None,
help=
"During running, it prepares a migration file from the state of the present data."
)
parser.add_option(
"-m",
"--migration",
dest="migration_file",
default=None,
help=
'Conveyance of previous migration file that was part of the previously run version.'
)
parser.add_option(
"-p",
"--path",
dest="path",
default=None,
help=
"Conveyance of a folder, which is added to the PATH variable in order that the link in the YAML configuration could be run on an outside python file."
)
parser.add_option("-d",
"--debug",
action='store_true',
default=False,
help="Debug mode, writes everything out as stdout.")
parser.add_option("-l",
"--limit",
help="Determines the number of elements for processing.")
parser.add_option(
"-o",
"--offset",
default=0,
help='Determines from which element should the processing start.')
parser.add_option(
"-s",
"--source",
help=
"If the configuration does not contain the route of the resource, it could be given here as well."
)
(options, args) = parser.parse_args(argv[1:])
if len(args) != 1:
parser.print_help()
sys.exit()
if options.path is not None:
absdirectory = os.path.abspath(options.path)
sys.path.append(absdirectory)
configparser = metl.configparser.ConfigParser(
metl.config.Config(args[0]),
debug=options.debug,
limit=options.limit,
offset=options.offset,
source_resource=options.source)
metl.manager.Manager(
configparser.getTarget(),
migration_resource=options.migration_file,
target_migration_resource=options.target_migration_file).run()
