import time
from functools import partial
from multiprocessing.pool import ThreadPool
from metasdk import MetaApp
META = MetaApp()
def my_thread_fn(bulk_log, job_item):
bulk_log.try_log_part()
# Бизнес логика
# работа с job_item
time.sleep(1)
def my_main_fn():
total = 125 * 2
thread_cnt = 2
bulk_log = META.bulk_log(u'Моя пачка', total, 1)
bulk_log.try_log_part()
all_data = range(1, total)
pool = ThreadPool(thread_cnt)
# Чтобы работать в многопоточном режиме с bulk_log вы
# должны передать его как аргумент вызываемой функции таким образом
func = partial(my_thread_fn, bulk_log)
results = pool.map(func, all_data)
from metasdk import MetaApp META = MetaApp(include_worker=False) log = META.log
from metasdk import MetaApp
META = MetaApp()
log = META.log
META.auth_user_id = 10191
result = META.MediaService.download("93ed70c9-2aa9-4fc6-971e-b41619c46f30")
print(u"result.content = %s" % str(result.content))
import logging
import pytest
from _pytest.logging import LogCaptureFixture
from metasdk import MetaApp
log = MetaApp().log
def assert_and_clear_caplog(caplog: LogCaptureFixture, asserted_msg: str,
asserted_context: dict):
"""Assert first (zero) record in caplog, than clear it"""
assert caplog.records[0].message == asserted_msg
assert caplog.records[0].msg == asserted_msg
assert caplog.records[0].context == asserted_context
caplog.clear()
def test_log_entity(caplog):
"""Test work with LOG_ENTITY on different log levels"""
with caplog.at_level(logging.INFO):
for log_level in [log.info, log.error]:
# Try log with context, without some set entities
log_level("info_message_0",
{"info_context_key_0": "info_context_value_0"})
assert_and_clear_caplog(
caplog, "info_message_0",
import os
from metasdk import MetaApp
META = MetaApp()
log = META.log
os.chdir(os.path.dirname(__file__))
__DIR__ = os.getcwd() + "/"
configuration = {
"download": {
# "skipHeaders": True, Если не нужны заголовки
"dbQuery": {
"command":
"SELECT * FROM products WHERE productname ilike '%Gula%' LIMIT 1000"
}
}
}
db = META.db("nw")
db.download_data(configuration,
output_file=__DIR__ + 'assets/out_products.tsv')
from metasdk import MetaApp
META = MetaApp(starter_api_url="http://s2.meta.vmc.loc:28341")
task_data = META.StarterService.submit("adptools.clear_logs", {})
print(u"task_data = %s" % str(task_data))
def optional_await_task_callback(task_info, is_finish):
print(u"task_info = %s" % str(task_info))
print(u"is_finish = %s" % str(is_finish))
result_task_info = META.StarterService.await_task(
task_data.get('taskId'), task_data.get('serviceId'),
optional_await_task_callback)
print(u"result_task_info = %s" % str(result_task_info))
import pytest
from metasdk import ApiProxyService, MetaApp
from metasdk.exceptions import ForbiddenError
api_proxy_service = ApiProxyService(MetaApp())
class MockResponse:
def __init__(self, json_data, status_code):
self.json_data = json_data
self.status_code = status_code
def json(self):
return self.json_data
forbidden_error_response = MockResponse(
{"error": {
"type": "ForbiddenError",
"text": "azaza"
}}, 200)
def mock_post(*_, **__):
return forbidden_error_response
def test_raise_business_api_proxy_business_errors_on_call_proxy(mocker):
mocker.patch('requests.post', new=mock_post)
with pytest.raises(ForbiddenError):
from metasdk import MetaApp
META = MetaApp(meta_url="http://localhost:8080")
META.ObjectLogService.log({
"userId": 10191,
"entityId": 2671,
"objectId": 53,
"code": "SET",
"form": "my_form", # optional
"value": { # optional
"foo": "bar",
"subFoo": {
"foo2": [1, 2, 3]
}
}
})
# А тут кроме лога запустится обработчик Шины событий, так как он есть именно на эту связку entity + code
# META.ObjectLogService.log({
# "userId": 10191,
# "entityId": 2830,
# "objectId": "7fb540bf-3cb3-4fa8-b11e-9a5a593d500b",
# "code": "SET",
# # "form": None,
# "form": "test",
# "value": {
# "a": 1
# }
# })
import base64
from metasdk import MetaApp
from metasdk.utils import pretty_json
META = MetaApp(meta_url="http://localhost:8080")
log = META.log
#
# Вы можете установить ID пользователя, от лица которого будут работать запросы
# Это полезно, когда вам надо сгенерировать приватный файл в фоновом режиме.
# Это user_id вы можете передать и прочитать из поля data в task
#
META.auth_user_id = 3503
YOUR_FILE_CONTENT_BASE64 = base64.b64encode(b'Custom user file').decode("utf-8")
# Получаете инстанс сервиса и делаете запрос к нему
result = META.MediaService.persist_one(
file_base64_content=YOUR_FILE_CONTENT_BASE64,
filename="req.txt",
extension="txt",
mime="plain/text"
)
print(u"result = %s" % result)
# Формат ответа стандартный для меты
first = result['rows'][0]
print(u"result['rows'][0]['url'] = %s" % first['url'])
print(u"first = %s" % first)
print(u"result = %s" % pretty_json(result))
import os
from metasdk import MetaApp
META = MetaApp()
log = META.log
os.chdir(os.path.dirname(os.path.abspath(__file__)))
__DIR__ = os.getcwd() + "/"
upload_file = open(__DIR__ + 'assets/load_data_sample.tsv', 'rb')
configuration = {
"load": {
"destinationTable": {
"schema": "public",
"table": "xxx_ya_stat"
},
"schema": {
"fields": [
{"name": "Date", "type": "DATE"},
{"name": "Clicks", "type": "LONG"},
{"name": "Cost", "type": "DECIMAL"},
{"name": "AdNetworkType", "type": "TEXT"},
]
}
}
}
db = META.db("meta_samples")
result = db.upload_data(upload_file, configuration)
from metasdk import MetaApp, StarterService
META = MetaApp()
starter = StarterService(META, META.db("garpun_main"),
"http://code.harpoon.lan:28341")
res = starter.submit("Notice.DeactivateOldNotices", {"foo": "bar"})
print(u"res = %s" % str(res))
import time
import threading
from metasdk import MetaApp
Meta = MetaApp()
lock = Meta.LockService.lock
TIME_TO_SLEEP = 20
TIMEOUT = 50
TTL = 50
KEY = "gogo"
WIDTH = 2
FIELD_TO_UNIQUE_DECORATOR = ["x"]
def lock_via_with(name):
with lock(timeout_in_sec=TIMEOUT,
ttl_in_sec=TTL,
key=KEY,
queue_width=WIDTH):
print("%s is running" % name)
time.sleep(TIME_TO_SLEEP)
print("%s is stop" % name)
@Meta.LockService.lock_decorator(timeout_in_sec=TIMEOUT,
ttl_in_sec=TTL,
key=KEY,
queue_width=WIDTH,
field_to_uniq=FIELD_TO_UNIQUE_DECORATOR)
from metasdk import MetaApp
from metasdk.utils import pretty_json
META = MetaApp(meta_url="https://meta.realweb.ru")
log = META.log
log.info('start')
try:
db_general = META.db("garpun_main")
camps = db_general.all("""
SELECT
"adHandsCampaignId" as id,
name,
"accountId" as account_id,
"campaignRemoteId",
"engine"
FROM global.campaigns
WHERE "virtualCampaignId" > 0
LIMIT 2
""")
for camp in camps:
storage_db = META.db("garpun_storage", camp['account_id'])
search_params = {"engine": camp['engine'], "camp_remote_id": camp['campaignRemoteId'],}
res = storage_db.one("""
SELECT count(1) as ads_cnt
FROM #shard.ads
INNER JOIN #shard."campaigns" c USING("campaignLocalId")
INNER JOIN #shard."virtualCampaigns" vc USING("virtualCampaignId")
WHERE vc.engine=:engine::"enum_engineType" AND c."campaignRemoteId"=:camp_remote_id
""", search_params)
camp.setdefault('counter', {})
import os
from metasdk import MetaApp
from datetime import datetime
from metasdk.utils import pretty_json
t1 = datetime.now()
META = MetaApp()
log = META.log
os.chdir(os.path.dirname(__file__))
__DIR__ = os.getcwd() + "/"
source_db = META.db("adplatform")
destination_db = META.db("iata")
source_tbl_schema = "iata"
source_tbl_name = "prices"
destination_tbl_schema = "public"
destination_tbl_name = "prices_copy"
OUTFILE = __DIR__ + 'assets/out_' + source_tbl_name + '.tsv'
configuration = {
"download": {
"dbQuery": {
"command": "SELECT * FROM " + source_tbl_schema + "." + source_tbl_name
}
import time
from metasdk import MetaApp
META = MetaApp()
# Самый простой вариант без знания об общем количестве элементов
bulk_log = META.bulk_log(u'Моя пачка')
for idx in range(100):
if bulk_log.try_log_part():
print(u"bulk_log.get_percent_done() = %s" %
str(bulk_log.get_percent_done()))
time.sleep(1)
bulk_log.finish()
# Стандартный вариант
total = 125
bulk_log = META.bulk_log(u'Моя пачка', total, 1)
for idx in range(total):
if bulk_log.try_log_part():
print(u"bulk_log.get_percent_done() = %s" %
str(bulk_log.get_percent_done()))
time.sleep(1)
bulk_log.finish()
# На частых но возможно долгих процессах
bulk_log = META.bulk_log(u'Моя пачка', total, 1)
for idx in range(total):
from metasdk import MetaApp
from datetime import datetime
t1 = datetime.now()
META = MetaApp()
log = META.log
META.auth_user_id = 11790
# Пример прогрева кеша
result = META.CacheService.warm_page_cache(
'88', '4993', None, None, {"stateParams": {
"firstLoading": True
}})
print(u"export_res = %s" % str(result))
import logging
from metasdk import MetaApp
META = MetaApp(debug=False)
log = META.log
e = Exception("Русский текст")
log.error("Моя ошибка", {"e": e})
log.set_entity('campaign_id', -1)
log.set_entity('test', True)
log.warning('Do warning log', {"count": 1, "mycontextParam": [1, 3, 4]})
log.info('Info log')
log.error('Info log')
logging.info('Default logging')
log.remove_entity('test')
log.info('Info log2')
from metasdk import MetaApp
from metasdk.utils import pretty_json
META = MetaApp()
log = META.log
#
# Db.one
#
db_adplatform = META.db("adplatform")
u = db_adplatform.one("""
SELECT id, name, info
FROM users
WHERE id = -1
LIMIT 1
""")
# Выдаст None
print(u"u = %s" % u)
#
# Db.query
#
db_adplatform = META.db("adplatform")
dr = db_adplatform.query("""
SELECT id, name, info
FROM users
WHERE name ILIKE 'Андре%'
ORDER BY name
LIMIT 1
""")
print(u"dr = %s" % pretty_json(dr))