def copy_files(args):
# Create the queue
queue = multiprocessing.JoinableQueue()
# Start the consumers
for x in range(PROCESSES):
consumer = Consumer(queue)
consumer.start()
# Look in the output directory
for root, dir_names, filenames in os.walk(args.product_dir):
LOGGER.debug('root: {0}, dir_names: {1}, filenames: {2}'.format(
root, dir_names, filenames))
for match in fnmatch.filter(dir_names,
'13B-266*calibrated_deepfield.ms'):
result_dir = join(root, match)
LOGGER.info('Queuing result_dir: {0}'.format(result_dir))
queue.put(
CopyTask(args.bucket, match, result_dir,
args.aws_access_key_id, args.aws_secret_access_key))
# Add a poison pill to shut things down
for x in range(PROCESSES):
queue.put(None)
# Wait for the queue to terminate
queue.join()
def process_entry(self, dn, entry):
try:
address = entry['cn'][0].decode("utf-8")
except KeyError:
try:
address = entry['dn'][0].decode("utf-8")
except KeyError:
return
else:
for field in entry:
if self.handler.check_field(field) is False:
continue
for value in entry[field]:
try:
decode_value = value.decode("utf-8")
except UnicodeDecodeError:
decode_value = value.decode('latin-1')
try:
if self.handler.execute(decode_value) is False:
continue
except IndexError:
continue
else:
LOGGER.debug(
'Address {} has wrong field: {}: {}'.format(
address, field, decode_value))
self.output_file.write(address + ' ' + field + ': ' +
decode_value + '\n')
def open_mdb_connection():
for host in MARIA_DB_CONFIGURATION['hosts']:
try:
connection = pymysql.connect(
user=MARIA_DB_CONFIGURATION['user'],
password=MARIA_DB_CONFIGURATION['password'],
database=MARIA_DB_CONFIGURATION['database'],
host=host,
cursorclass=pymysql.cursors.DictCursor,
autocommit=True,
)
except:
if host == MARIA_DB_CONFIGURATION['hosts'][-1]:
log_str = "Failed to connect to any MariaDB host"
LOGGER.exception(log_str, exc_info=True)
raise DBError(log_str)
else:
LOGGER.warning(
'Failed to connect to MariaDB on host {}. Trying next host.'
.format(host))
else:
if connection.open:
LOGGER.debug(
'mariadb connection to host {} successful.'.format(host))
return connection
else:
err_str = 'Connection to MariaDB failed.'
LOGGER.error(err_str)
raise DBError(err_str)
def copy_files(args):
# Create the queue
queue = multiprocessing.JoinableQueue()
# Start the consumers
for x in range(PROCESSES):
consumer = Consumer(queue)
consumer.start()
# Look in the output directory
for root, dir_names, filenames in os.walk(args.product_dir):
LOGGER.debug('root: {0}, dir_names: {1}, filenames: {2}'.format(root, dir_names, filenames))
for match in fnmatch.filter(dir_names, '13B-266*calibrated_deepfield.ms'):
result_dir = join(root, match)
LOGGER.info('Queuing result_dir: {0}'.format(result_dir))
queue.put(
CopyTask(
args.bucket,
match,
result_dir,
args.aws_access_key_id,
args.aws_secret_access_key
)
)
# Add a poison pill to shut things down
for x in range(PROCESSES):
queue.put(None)
# Wait for the queue to terminate
queue.join()
def check_entry_mariadb(address, mariadb_connection, query):
LOGGER.debug('Checking account: {}'.format(remove_01019(address)))
cursor = mariadb_connection.cursor()
query_string = str(query).format(table=MARIA_DB_CONFIGURATION['table'],
address=remove_01019(address))
LOGGER.debug('Mariadb query: ' + query_string)
mariadb_hits = cursor.execute(query_string)
if mariadb_hits == 0:
LOGGER.debug('Account not found in mariadb during policy period.')
return 'Not found'
else:
result = cursor.fetchone()
LOGGER.debug('MariaDB first result: \n{}'.format(result))
return result['ts'].strftime("%m/%d/%Y, %H:%M:%S")
def process_entry(self, dn, entry):
if entry['objectClass'][-1].decode("utf-8") == 'mailaccount':
object_class = 'mailaccount'
address = entry['cn'][0].decode("utf-8")
if 'forwardto' in entry:
LOGGER.debug('Skipping alias account {}'.format(address))
return
else:
LOGGER.debug('Skipping object of class {}'.format(
entry['objectClass'][-1].decode("utf-8")))
return
if 'lock' in entry:
lock = entry['lock'][0].decode("utf-8")
if lock == 'submit':
LOGGER.info('Address {} has lock = submit!'.format(
address, self.mariadb_connection))
locking_date = check_entry_mariadb(address,
self.mariadb_connection,
self.query)
self.output_file.write(address + ' ' + object_class +
' locking date in MariaDB: ' +
locking_date + '\n')
def name_from_text(text):
return re.sub(r""",- !@#$%^&*;:."(')//\\""", '',
text).replace(' ', '').lower()[:250]
def random_name(x=10):
try:
return "".join([random.choice(string.letters) for i in range(x)])
except:
return "".join([random.choice(string.ascii_letters) for i in range(x)])
if __name__ == "__main__":
logger = LOGGER('TALK', 'INFO')
p = CONFIGURATION()
logger.debug('config read')
# versions
try:
m.logger.info('gtts version: ' + str(gtts.__version__))
except:
pass
if len(sys.argv) > 1:
text = ' '.join([i for i in sys.argv[1:] if i != '-d'])
if text == '': text = 'Hello world'
else:
text = "hello world"
Speak(text)
