def _setup_tables(account_name, account_key, table_name, batch_size=100, max_num=1000000):
table_service = TableService(account_name, account_key)
table_service.create_table(table_name)
partitions = defaultdict(list)
for num in range(1, max_num + 1):
partitions[('%03d' % num)[:3]].append(str(num))
for partition, nums in partitions.items():
for batch_num, batch in enumerate(_grouper(nums, batch_size), start=1):
table_batch = TableBatch()
for num in filter(None, batch):
table_batch.insert_entity({
'PartitionKey': partition,
'RowKey': num,
'value': str(uuid4()),
})
table_service.commit_batch(table_name, table_batch)
print('Done with partition %s, batch %d' % (partition, batch_num))
except:
own_posts_ratio =0
reposts_count, reposts = number_of_reposts(api, friend.user_id)
reposts_hIndex = hIndex(reposts)
likes_count, scores =number_of_likes(api, friend.user_id)
likes_hIndex = hIndex(scores)
friends_info ={'PartitionKey': 'my_friends1', 'RowKey': str(k), 'first_name': friend.first_name, 'last_name': friend.last_name,
'user_id': friend.user_id, 'friends_count': friends_count, 'groups_count': groups_count, 'posts_count': posts_count,
'own_posts': own_posts_count, 'followers_count': followers_count, 'own_posts_ratio': own_posts_ratio,
'likes_count': likes_count, 'likes_hIndex',likes_hIndex, 'reposts_count', reposts_count, 'reposts_hIndex', reposts_hIndex}
batch.insert_or_replace_entity(friends_info)
if k%10==0:
table_service.commit_batch('MyVkApp', batch)
batch =TableBatch()
print("#: ",k)
print("Id: ",friend.user_id)
print("Имя: ",friend.first_name)
print("Фамилия: ",friend.last_name)
print('Количество друзей: ', friends_count)
print('Количество групп: ', groups_count)
print('Количество постов: ', posts_count)
print('Количество своих постов: ', own_posts_count)
print('Количество подписчиков: ', followers_count)
print('Соотношение своих постов со всеми постами: ', own_posts_ratio)
print('Количество лайков: ', likes_count)
print('hIndex likes: ',likes_hIndex)
print('Количество репостов: ', reposts_count)
print('hIndex репостов: ', reposts_hIndex)
class TableStorageHandler(logging.Handler):
"""
Handler class which writes log messages to a Azure Storage table.
"""
MAX_BATCH_SIZE = 100
def __init__(self,
account_name=None,
account_key=None,
protocol='https',
table='logs',
batch_size=0,
extra_properties=None,
partition_key_formatter=None,
row_key_formatter=None,
is_emulated=False,
):
"""
Initialize the handler.
"""
logging.Handler.__init__(self)
self.service = TableService(account_name=account_name,
account_key=account_key,
is_emulated=is_emulated,
protocol=protocol)
self.meta = {'hostname': gethostname(), 'process': os.getpid()}
self.table = _formatName(table, self.meta)
self.ready = False
self.rowno = 0
if not partition_key_formatter:
# default format for partition keys
fmt = '%(asctime)s'
datefmt = '%Y%m%d%H%M'
partition_key_formatter = logging.Formatter(fmt, datefmt)
self.partition_key_formatter = partition_key_formatter
if not row_key_formatter:
# default format for row keys
fmt = '%(asctime)s%(msecs)03d-%(hostname)s-%(process)d-%(rowno)02d'
datefmt = '%Y%m%d%H%M%S'
row_key_formatter = logging.Formatter(fmt, datefmt)
self.row_key_formatter = row_key_formatter
# extra properties and formatters for them
self.extra_properties = extra_properties
if extra_properties:
self.extra_property_formatters = {}
self.extra_property_names = {}
for extra in extra_properties:
if _PY3:
f = logging.Formatter(fmt=extra, style=extra[0])
else:
f = logging.Formatter(fmt=extra)
self.extra_property_formatters[extra] = f
self.extra_property_names[extra] = self._getFormatName(extra)
# the storage emulator doesn't support batch operations
if batch_size <= 1 or is_emulated:
self.batch = None
else:
self.batch = TableBatch()
if batch_size > TableStorageHandler.MAX_BATCH_SIZE:
self.batch_size = TableStorageHandler.MAX_BATCH_SIZE
else:
self.batch_size = batch_size
if self.batch:
self.current_partition_key = None
def _copyLogRecord(self, record):
copy = logging.makeLogRecord(record.__dict__)
copy.exc_info = None
copy.exc_text = None
if _PY3:
copy.stack_info = None
return copy
def _getFormatName(self, extra):
name = extra
style = extra[0]
if style == '%':
name = extra[2:extra.index(')')]
elif _PY3:
if style == '{':
name = next(string.Formatter().parse(extra))[1]
elif style == '$':
name = extra[1:]
if name.startswith('{'):
name = name[1:-1]
return name
def emit(self, record):
"""
Emit a record.
Format the record and send it to the specified table.
"""
try:
if not self.ready:
self.service.create_table(self.table)
self.ready = True
# generate partition key for the entity
record.hostname = self.meta['hostname']
copy = self._copyLogRecord(record)
partition_key = self.partition_key_formatter.format(copy)
# ensure entities in the batch all have the same patition key
if self.batch:
if self.current_partition_key is not None:
if partition_key != self.current_partition_key:
self.flush()
self.current_partition_key = partition_key
# add log message and extra properties to the entity
entity = {}
if self.extra_properties:
for extra in self.extra_properties:
formatter = self.extra_property_formatters[extra]
name = self.extra_property_names[extra]
entity[name] = formatter.format(copy)
entity['message'] = self.format(record)
# generate row key for the entity
copy.rowno = self.rowno
row_key = self.row_key_formatter.format(copy)
# add entitiy to the table
entity['PartitionKey'] = partition_key
entity['RowKey'] = row_key
if not self.batch:
self.service.insert_or_replace_entity(self.table, entity)
else:
self.batch.insert_or_replace_entity(entity)
# commit the ongoing batch if it reaches the high mark
self.rowno += 1
if self.rowno >= self.batch_size:
self.flush()
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
def flush(self):
"""
Ensure all logging output has been flushed.
"""
if self.batch and self.rowno > 0:
try:
self.service.commit_batch(self.table, self.batch)
finally:
self.rowno = 0
self.batch = TableBatch()
def setFormatter(self, fmt):
"""
Set the message formatter.
"""
super(TableStorageHandler, self).setFormatter(fmt)
if self.extra_properties:
logging._acquireLock()
try:
for extra in self.extra_property_formatters.values():
extra.converter = fmt.converter
extra.datefmt = fmt.datefmt
if _PY3:
extra.default_time_format = fmt.default_time_format
extra.default_msec_format = fmt.default_msec_format
finally:
logging._releaseLock()
def setPartitionKeyFormatter(self, fmt):
"""
Set the partition key formatter.
"""
self.partition_key_formatter = fmt
def setRowKeyFormatter(self, fmt):
"""
Set the row key formatter.
"""
self.row_key_formatter = fmt
class TableStorageHandler(logging.Handler):
"""
Handler class which writes log messages to a Azure Storage table.
"""
MAX_BATCH_SIZE = 100
def __init__(self,
account_name=None,
account_key=None,
protocol='https',
table='logs',
batch_size=0,
extra_properties=None,
partition_key_formatter=None,
row_key_formatter=None,
):
"""
Initialize the handler.
"""
logging.Handler.__init__(self)
self.service = TableService(account_name=account_name,
account_key=account_key,
protocol=protocol)
self.meta = {'hostname': gethostname(), 'process': os.getpid()}
self.table = _formatName(table, self.meta)
self.ready = False
self.rowno = 0
if not partition_key_formatter:
# default format for partition keys
fmt = '%(asctime)s'
datefmt = '%Y%m%d%H%M'
partition_key_formatter = logging.Formatter(fmt, datefmt)
self.partition_key_formatter = partition_key_formatter
if not row_key_formatter:
# default format for row keys
fmt = '%(asctime)s%(msecs)03d-%(hostname)s-%(process)d-%(rowno)02d'
datefmt = '%Y%m%d%H%M%S'
row_key_formatter = logging.Formatter(fmt, datefmt)
self.row_key_formatter = row_key_formatter
# extra properties and formatters for them
self.extra_properties = extra_properties
if extra_properties:
self.extra_property_formatters = {}
self.extra_property_names = {}
for extra in extra_properties:
if _PY3:
f = logging.Formatter(fmt=extra, style=extra[0])
else:
f = logging.Formatter(fmt=extra)
self.extra_property_formatters[extra] = f
self.extra_property_names[extra] = self._getFormatName(extra)
# the storage emulator doesn't support batch operations
if batch_size <= 1 or self.service.use_local_storage:
self.batch = False
else:
self.batch = True
if batch_size > TableStorageHandler.MAX_BATCH_SIZE:
self.batch_size = TableStorageHandler.MAX_BATCH_SIZE
else:
self.batch_size = batch_size
if self.batch:
self.current_partition_key = None
def _copyLogRecord(self, record):
copy = logging.makeLogRecord(record.__dict__)
copy.exc_info = None
copy.exc_text = None
if _PY3:
copy.stack_info = None
return copy
def _getFormatName(self, extra):
name = extra
style = extra[0]
if style == '%':
name = extra[2:extra.index(')')]
elif _PY3:
if style == '{':
name = next(string.Formatter().parse(extra))[1]
elif style == '$':
name = extra[1:]
if name.startswith('{'):
name = name[1:-1]
return name
def emit(self, record):
"""
Emit a record.
Format the record and send it to the specified table.
"""
try:
if not self.ready:
self.service.create_table(self.table)
if self.batch:
self.service.begin_batch()
self.ready = True
# generate partition key for the entity
record.hostname = self.meta['hostname']
copy = self._copyLogRecord(record)
partition_key = self.partition_key_formatter.format(copy)
# ensure entities in the batch all have the same patition key
if self.batch:
if self.current_partition_key is not None:
if partition_key != self.current_partition_key:
self.flush()
self.current_partition_key = partition_key
# add log message and extra properties to the entity
entity = {}
if self.extra_properties:
for extra in self.extra_properties:
formatter = self.extra_property_formatters[extra]
name = self.extra_property_names[extra]
entity[name] = formatter.format(copy)
entity['message'] = self.format(record)
# generate row key for the entity
copy.rowno = self.rowno
row_key = self.row_key_formatter.format(copy)
# add entitiy to the table
self.service.insert_or_replace_entity(self.table,
partition_key,
row_key,
entity)
# commit the ongoing batch if it reaches the high mark
if self.batch:
self.rowno += 1
if self.rowno >= self.batch_size:
self.flush()
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
def flush(self):
"""
Ensure all logging output has been flushed.
"""
if self.batch and self.rowno > 0:
try:
self.service.commit_batch()
finally:
self.rowno = 0
self.service.begin_batch()
def setFormatter(self, fmt):
"""
Set the message formatter.
"""
super(TableStorageHandler, self).setFormatter(fmt)
if self.extra_properties:
logging._acquireLock()
try:
for extra in self.extra_property_formatters.values():
extra.converter = fmt.converter
extra.datefmt = fmt.datefmt
if _PY3:
extra.default_time_format = fmt.default_time_format
extra.default_msec_format = fmt.default_msec_format
finally:
logging._releaseLock()
def setPartitionKeyFormatter(self, fmt):
"""
Set the partition key formatter.
"""
self.partition_key_formatter = fmt
def setRowKeyFormatter(self, fmt):
"""
Set the row key formatter.
"""
self.row_key_formatter = fmt
def insert_options_azure(optionChain):
# receives the optionChain object containing all options for all expiration dates
# for the selected symbol. inserts rows into the database options table for
# each option. Performs a db INSERT statement. If the row already exists,
# the database will generate an invalid key error to prevent the row from
# being duplicated in the table. In this case, the error is ignored.
#
account = ut.get_azure_account()
table_service = None
table_name = ut.TABLE_NAME_OPTIONS
try:
if config.IS_EMULATED:
table_service = TableService(is_emulated=True)
else:
table_service = TableService(
account_name=config.STORAGE_ACCOUNT_NAME,
account_key=config.STORAGE_ACCOUNT_KEY)
if not table_service.exists(table_name):
# create the table
try:
table_service.create_table(table_name)
except Exception as err:
print('Error creating table, ' + table_name +
'check if it already exists')
lg.error(
'Tried and failed to create the table for the symbols. Program terminating...'
)
exit()
batch = TableBatch()
batchCount = 0
rowCount = 0
for o in optionChain.options:
rowCount += 1
if rowCount > 100:
# Azure restricts the batch size to a max of a hundred entries. Since we're at our
# limit, we'll commit these and start a new batch
table_service.commit_batch(table_name, batch)
batch = TableBatch()
rowCount = 1
batchCount += 1
option = Entity()
option.PartitionKey = o.PartitionKey
# rowkey comprises the concatination of symbols to ensure the key is unique for the symbol.
# we'll use the callPut, optionDate, expirationDate, and strike price. Dates will be in format yyyymmdd
option.RowKey = o.RowKey
option.OptionDate = o.optionDate # dates are already cast as Entity Property with an aware date value
option.Expiration = o.expiration
option.CallPut = o.callPut
option.Strike = o.strike
option.Bid = o.bid
option.Ask = o.ask
option.LastPrice = o.lastPrice
option.Volume = o.volume
option.OpenInterest = o.openInterest
option.IV = o.impliedVolatility
option.StockPrice = o.stockPrice
batch.insert_entity(option)
table_service.commit_batch(table_name, batch)
except Exception as e:
print('Error adding option ' + symbol + '. Error is: ', e)
lg.error('Error adding rows to the options table')
def insert_historical_option_data(rows):
account = ut.get_azure_account()
table_service = None
table_name = 'optionSandbox' # ut.TABLE_NAME_OPTIONS
try:
if config.IS_EMULATED:
table_service = TableService(is_emulated=True)
else:
table_service = TableService(
account_name=config.STORAGE_ACCOUNT_NAME,
account_key=config.STORAGE_ACCOUNT_KEY)
if not table_service.exists(table_name):
# create the table
try:
table_service.create_table(table_name)
except Exception as err:
print('Error creating table, ' + table_name +
'check if it already exists')
lg.error(
'Tried and failed to create the table for the symbols. Program terminating...'
)
exit()
batch = TableBatch()
batchCount = 0
rowCount = 0
for row in rows:
option = Entity()
callPut = str(row[5]).strip()
optionDate = row[7]
expiration = row[6]
strike = float(row[8])
option.PartitionKey = str(row[0]).strip()
# rowkey comprises the concatination of symbols to ensure the key is unique for the symbol.
#option.RowKey = callPut + optionDate.strftime('%Y%m%d') + expiration.strftime('%Y%m%d') + str(strike)
optionDateYYYY = optionDate[-4:]
optionDateMM = optionDate[:2]
optionDateDD = optionDate[3:5]
optionDateRowKey = optionDateYYYY + optionDateMM + optionDateDD
expDateYYYY = expiration[-4:]
expDateMM = expiration[:2]
expDateDD = expiration[3:5]
expDateRowKey = expDateYYYY + expDateMM + expDateDD
option.RowKey = callPut + optionDateRowKey + expDateRowKey + str(
strike)
option.OptionDate = ut.historicalLoadDates[
optionDate] # ut.date_for_azure(optionDate)
option.Expiration = ut.historicalLoadDates[
expiration] # ut.date_for_azure(expiration)
option.CallPut = callPut
option.Strike = strike
option.Bid = float(row[10])
option.Ask = float(row[11])
option.LastPrice = float(row[9])
option.Volume = float(row[12])
option.OpenInterest = int(row[13])
option.StockPrice = float(row[1])
batch.insert_or_replace_entity(option)
table_service.commit_batch(table_name, batch)
except Exception as e:
print('Error importing option ' + symbol + '. Error is: ', e)
lg.error('Error importing rows to the options table')
task10 = {
'PartitionKey': 'tasksSeattle',
'RowKey': '10',
'description': 'Go grocery shopping',
'priority': 400
}
task11 = {
'PartitionKey': 'tasksSeattle',
'RowKey': '11',
'description': 'Clean the bathroom',
'priority': 100
}
table_service.begin_batch()
table_service.insert_entity('tasktable', task10)
table_service.insert_entity('tasktable', task11)
table_service.commit_batch()
task = table_service.get_entity('tasktable', 'tasksSeattle', '1')
print(task.description)
print(task.priority)
tasks = table_service.query_entities('tasktable',
"PartitionKey eq 'tasksSeattle'")
for task in tasks:
print(task.description)
print(task.priority)
tasks = table_service.query_entities('tasktable',
"PartitionKey eq 'tasksSeattle'",
'description')
for task in tasks:
#
task = {'description' : 'Take out the garbage', 'priority' : 250}
table_service.update_entity('tasktable', 'tasksSeattle', '1', task)
#
task = {'description' : 'Take out the garbage again', 'priority' : 250}
table_service.insert_or_replace_entity('tasktable', 'tasksSeattle', '1', task)
task = {'description' : 'Buy detergent', 'priority' : 300}
table_service.insert_or_replace_entity('tasktable', 'tasksSeattle', '3', task)
task10 = {'PartitionKey': 'tasksSeattle', 'RowKey': '10', 'description' : 'Go grocery shopping', 'priority' : 400}
task11 = {'PartitionKey': 'tasksSeattle', 'RowKey': '11', 'description' : 'Clean the bathroom', 'priority' : 100}
table_service.begin_batch()
table_service.insert_entity('tasktable', task10)
table_service.insert_entity('tasktable', task11)
table_service.commit_batch()
task = table_service.get_entity('tasktable', 'tasksSeattle', '1')
print(task.description)
print(task.priority)
tasks = table_service.query_entities('tasktable', "PartitionKey eq 'tasksSeattle'")
for task in tasks:
print(task.description)
print(task.priority)
tasks = table_service.query_entities('tasktable', "PartitionKey eq 'tasksSeattle'", 'description')
for task in tasks:
print(task.description)
table_service.delete_entity('tasktable', 'tasksSeattle', '1')
