def __init__(self, elem, import_file, transform_only=False):
Master.__init__(self, elem)
self.additional_code_sid = Master.process_null(elem.find("sid"))
self.additional_code_type_id = Master.process_null(
elem.find("additionalCodeType/additionalCodeTypeId"))
self.additional_code = Master.process_null(
elem.find("additionalCodeCode"))
self.validity_start_date = Master.process_date(
elem.find("validityStartDate"))
self.validity_end_date = Master.process_date(
elem.find("validityEndDate"))
operation_date = g.app.get_timestamp()
g.app.print_only(
"Running operation {0} on additional code {1}.".format(
self.operation, str(self.additional_code_sid)))
# Insert the additional code
if transform_only is False:
sql = """
insert into additional_codes_oplog
(additional_code_sid, additional_code_type_id, additional_code, validity_start_date, validity_end_date,
operation, operation_date, created_at, filename)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
"""
params = [
self.additional_code_sid, self.additional_code_type_id,
self.additional_code, self.validity_start_date,
self.validity_end_date, self.operation, operation_date,
operation_date, import_file
]
d = Database()
d.run_query(sql, params)
# Delete any additional code descriptions
sql = """
delete from additional_code_descriptions_oplog
where additional_code_sid = %s
"""
params = [self.additional_code_sid]
d = Database()
d.run_query(sql, params)
# Delete any additional code description periods
sql = """
delete from additional_code_description_periods_oplog
where additional_code_sid = %s
"""
params = [self.additional_code_sid]
d = Database()
d.run_query(sql, params)
# Create new additional code description periods
for elem in elem.findall('.//additionalCodeDescriptionPeriod'):
AdditionalCodeDescriptionPeriod(elem, self.additional_code_sid,
self.additional_code_type_id,
self.additional_code,
import_file)
def __init__(self, elem, measure_sid, import_file, transform_only=False):
Master.__init__(self, elem)
self.measure_sid = measure_sid
self.measure_condition_sid = Master.process_null(elem.find("sid"))
self.condition_code = Master.process_null(
elem.find("measureConditionCode/conditionCode"))
self.component_sequence_number = Master.process_null(
elem.find("conditionSequenceNumber"))
self.condition_duty_amount = Master.process_null(
elem.find("conditionDutyAmount"))
self.condition_monetary_unit_code = Master.process_null(
elem.find("monetaryUnit/monetaryUnitCode"))
self.condition_measurement_unit_code = Master.process_null(
elem.find("measurementUnit/measurementUnitCode"))
self.condition_measurement_unit_qualifier_code = Master.process_null(
elem.find("measurementUnitQualifier/measurementUnitQualifierCode"))
self.action_code = Master.process_null(
elem.find("measureAction/actionCode"))
self.certificate_type_code = Master.process_null(
elem.find("certificate/certificateType/certificateTypeCode"))
self.certificate_code = Master.process_null(
elem.find("certificate/certificateCode"))
operation_date = g.app.get_timestamp()
# Insert the measure condition
if transform_only is False:
sql = """
insert into measure_conditions_oplog
(measure_sid, measure_condition_sid, condition_code,
component_sequence_number, condition_duty_amount, condition_monetary_unit_code,
condition_measurement_unit_qualifier_code, action_code, certificate_type_code, certificate_code,
operation, operation_date, created_at, filename)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
"""
params = [
self.measure_sid, self.measure_condition_sid,
self.condition_code, self.component_sequence_number,
self.condition_duty_amount, self.condition_monetary_unit_code,
self.condition_measurement_unit_qualifier_code,
self.action_code, self.certificate_type_code,
self.certificate_code, self.operation, operation_date,
operation_date, import_file
]
d = Database()
d.run_query(sql, params)
# Delete any measure condition components
sql = """delete from measure_condition_components_oplog where measure_condition_sid = %s"""
params = [self.measure_condition_sid]
d = Database()
d.run_query(sql, params)
# Create new measure condition components
for elem1 in elem.findall('.//measureConditionComponent'):
MeasureConditionComponent(elem1, self.measure_condition_sid,
import_file)
def __init__(self, elem, import_file, transform_only=False):
Master.__init__(self, elem)
self.measure_type_id = Master.process_null(elem.find("measureTypeId"))
self.measure_component_applicable_code = Master.process_null(
elem.find("measureComponentApplicableCode"))
self.measure_explosion_level = Master.process_null(
elem.find("measureExplosionLevel"))
self.order_number_capture_code = Master.process_null(
elem.find("orderNumberCaptureCode"))
self.origin_dest_code = Master.process_null(
elem.find("originDestCode"))
self.priority_code = Master.process_null(elem.find("priorityCode"))
self.trade_movement_code = Master.process_null(
elem.find("tradeMovementCode"))
self.measure_type_series_id = Master.process_null(
elem.find("measureTypeSeries/measureTypeSeriesId"))
self.validity_start_date = Master.process_date(
elem.find("validityStartDate"))
self.validity_end_date = Master.process_date(
elem.find("validityEndDate"))
self.application_code = Master.process_null(
elem.find("applicationCode"))
operation_date = g.app.get_timestamp()
# Insert the measure type
if transform_only is False:
sql = """
insert into measure_types_oplog
(measure_type_id, validity_start_date, validity_end_date,
measure_component_applicable_code, measure_explosion_level, order_number_capture_code,
origin_dest_code, priority_code, trade_movement_code, measure_type_series_id,
operation, operation_date, created_at, filename)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
"""
params = [
self.measure_type_id, self.validity_start_date,
self.validity_end_date, self.measure_component_applicable_code,
self.measure_explosion_level, self.order_number_capture_code,
self.origin_dest_code, self.priority_code,
self.trade_movement_code, self.measure_type_series_id,
self.operation, operation_date, operation_date, import_file
]
d = Database()
d.run_query(sql, params)
# Delete any measure descriptions
sql = """
delete from measure_type_descriptions_oplog
where measure_type_id = %s
"""
params = [self.measure_type_id]
d = Database()
d.run_query(sql, params)
# Create new measure type descriptions
for elem in elem.findall('.//measureTypeDescription'):
MeasureTypeDescription(elem, self.measure_type_id, import_file)
def __init__(self, elem, import_file, transform_only=False):
Master.__init__(self, elem)
self.certificate_type_code = Master.process_null(
elem.find("certificateType/certificateTypeCode"))
self.certificate_code = Master.process_null(
elem.find("certificateCode"))
self.validity_start_date = Master.process_date(
elem.find("validityStartDate"))
self.validity_end_date = Master.process_date(
elem.find("validityEndDate"))
operation_date = g.app.get_timestamp()
g.app.print_only("Running operation {0} on certificate {1}.".format(
self.operation, str(self.certificate_code)))
# Insert the certificate
if transform_only is False:
sql = """
insert into certificates_oplog
(certificate_type_code, certificate_code, validity_start_date, validity_end_date,
operation, operation_date, created_at, filename)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
"""
params = [
self.certificate_type_code, self.certificate_code,
self.validity_start_date, self.validity_end_date,
self.operation, operation_date, operation_date, import_file
]
d = Database()
d.run_query(sql, params)
# Delete any certificate descriptions
sql = """
delete from certificate_descriptions_oplog
where certificate_type_code = %s and certificate_code = %s
"""
params = [self.certificate_type_code, self.certificate_code]
d = Database()
d.run_query(sql, params)
# Delete any certificate description periods
sql = """
delete from certificate_description_periods_oplog
where certificate_type_code = %s and certificate_code = %s
"""
params = [self.certificate_type_code, self.certificate_code]
d = Database()
d.run_query(sql, params)
# Create new certificate description periods
for elem in elem.findall('.//certificateDescriptionPeriod'):
CertificateDescriptionPeriod(elem, self.certificate_type_code,
self.certificate_code,
import_file)
def export(target,
type='target',
db=None,
alive=False,
limit=None,
path=None,
format='csv',
show=False):
"""
OneForAll export from database module
Example:
python3 dbexport.py --target name --format csv --dir= ./result.csv
python3 dbexport.py --db result.db --target name --show False
python3 dbexport.py --target table_name --tb True --show False
Note:
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
--path Result directory (default directory is ./results)
:param str target: Table to be exported
:param str type: Type of target
:param str db: Database path to be exported (default ./results/result.sqlite3)
:param bool alive: Only export the results of alive subdomains (default False)
:param str limit: Export limit (default None)
:param str format: Result format (default csv)
:param str path: Result directory (default None)
:param bool show: Displays the exported data in terminal (default False)
"""
if type == 'target':
database = Database(db)
domains = utils.get_domains(target)
datas = []
if domains:
for domain in domains:
table_name = domain_to_table(domain)
rows = database.export_data(table_name, alive, limit)
if rows is None:
continue
data = export_data(format, path, rows, show, table_name,
target)
datas.extend(data)
database.close()
if len(domains) > 1:
utils.export_all(alive, format, path, datas)
elif type == 'table':
database = Database(db)
rows = database.export_data(target, alive, limit)
data = export_data(format, path, rows, show, target, target)
database.close()
return data
def __init__(self, elem, import_file, transform_only=False):
Master.__init__(self, elem)
self.descriptions = []
self.footnote_type_id = Master.process_null(
elem.find("footnoteType/footnoteTypeId"))
self.footnote_id = Master.process_null(elem.find("footnoteId"))
self.validity_start_date = Master.process_date(
elem.find("validityStartDate"))
self.validity_end_date = Master.process_date(
elem.find("validityEndDate"))
operation_date = g.app.get_timestamp()
# Insert the footnote
if transform_only is False:
sql = """
insert into footnotes_oplog
(footnote_type_id, footnote_id, validity_start_date, validity_end_date,
operation, operation_date, created_at, filename)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
"""
params = [
self.footnote_type_id, self.footnote_id,
self.validity_start_date, self.validity_end_date,
self.operation, operation_date, operation_date, import_file
]
d = Database()
d.run_query(sql, params)
# Delete any footnote descriptions
sql = """
delete from footnote_descriptions_oplog
where footnote_type_id = %s and footnote_id = %s
"""
params = [self.footnote_type_id, self.footnote_id]
d = Database()
d.run_query(sql, params)
# Delete any footnote description periods
sql = """
delete from footnote_description_periods_oplog
where footnote_type_id = %s and footnote_id = %s
"""
params = [self.footnote_type_id, self.footnote_id]
d = Database()
d.run_query(sql, params)
# Create new footnote description periods
for elem in elem.findall('.//footnoteDescriptionPeriod'):
FootnoteDescriptionPeriod(elem, self.footnote_type_id,
self.footnote_id, import_file)
def __init__(self,
elem,
quota_definition_sid,
import_file,
transform_only=False):
Master.__init__(self, elem)
self.quota_definition_sid = quota_definition_sid
self.sub_quota_definition_sid = Master.process_null(
elem.find("subQuotaDefinition/sid"))
self.relation_type = Master.process_null(elem.find("relationType"))
self.coefficient = Master.process_null(elem.find("coefficient"))
operation_date = g.app.get_timestamp()
g.app.print_only(
"Running operation {0} on quota association {1}.".format(
self.operation, str(self.quota_definition_sid)))
if transform_only is False:
sql = """
insert into quota_associations_oplog
(main_quota_definition_sid, sub_quota_definition_sid, relation_type, coefficient,
operation, operation_date, created_at, filename)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
"""
params = [
self.quota_definition_sid, self.sub_quota_definition_sid,
self.relation_type, self.coefficient, self.operation,
operation_date, operation_date, import_file
]
d = Database()
d.run_query(sql, params)
def test_03_show_unshipped_order(self):
"""查看未发货订单"""
data={"session":self.setUp(),"type":"await_ship","pagination":{"count":10,"page":1}}
response=To_be_shipped().to_be_shipped(data)
unship_order_ids=get_keywords(response,"order_id")
print(unship_order_ids)
"""连接数据库"""
database = Database(host="ecshop.itsoso.cn", user="******", password="******", database="ecshop",
charset='utf8', port=3306)
sql = "select * from ecs_order_info where user_id=11402 and shipping_status=0"
order_id_db = database.readall(sql=sql)
print(order_id_db)
"""在数据库中获取取消订单的id"""
db_orderid = get_keywords(order_id_db, "order_id")
print(db_orderid)
"""断言"""
"""判断未发货的id是否存在于数据库取消订单的id中"""
if unship_order_ids:
for unship_order_id in unship_order_ids:
if unship_order_id in db_orderid:
self.assertTrue(True)
else:
self.assertTrue(False)
else:
print("未发货订单为零")
async def status(self, ctx):
if ctx.invoked_subcommand is None:
db = Database()
embed = discord.Embed(title='Bot Status')
counts = db.getCountEstimates()
status = db.getStatus()
embed.add_field(name="Bot Name", value="[Shinigami]", inline=False)
embed.add_field(
name="Bot Uptime",
value=
f"{humanize.naturaldelta(datetime.now() - status['botStartTime'])}",
inline=False)
embed.add_field(name="Current tracked game deals",
value=f"{counts['gamedeals']}",
inline=False)
embed.add_field(name="Current tracked game cracks",
value=f"{counts['cracks']}",
inline=False)
embed.add_field(name="Current tracked game repacks",
value=f"{counts['repacks']}",
inline=False)
embed.add_field(
name="Bot Source",
value=
f"[https://github.com/santoshpanna/Discord-Bot](github.com)",
inline=False)
await ctx.send(embed=embed)
def __init__(self,
elem,
quota_definition_sid,
import_file,
transform_only=False):
Master.__init__(self, elem)
self.quota_suspension_period_sid = Master.process_null(
elem.find("sid"))
self.quota_definition_sid = quota_definition_sid
self.suspension_start_date = Master.process_date(
elem.find("suspensionStartDate"))
self.suspension_end_date = Master.process_date(
elem.find("suspensionEndDate"))
self.description = Master.process_null(elem.find("description"))
operation_date = g.app.get_timestamp()
if transform_only is False:
sql = """
insert into quota_suspension_periods_oplog
(quota_suspension_period_sid, quota_definition_sid,
suspension_start_date, suspension_end_date, description,
operation, operation_date, created_at, filename)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
"""
params = [
self.quota_suspension_period_sid, self.quota_definition_sid,
self.suspension_start_date, self.suspension_end_date,
self.description, self.operation, operation_date,
operation_date, import_file
]
d = Database()
d.run_query(sql, params)
def deal_db(self):
"""
收集任务完成时对数据库进行处理
"""
db = Database()
db.deal_table(self.domain, self.origin_table)
db.close()
def __init__(self, elem, measure_sid, import_file, transform_only=False):
Master.__init__(self, elem)
self.measure_sid = measure_sid
self.excluded_geographical_area = Master.process_null(
elem.find("geographicalArea/geographicalAreaId"))
self.geographical_area_sid = Master.process_null(
elem.find("geographicalArea/sid"))
operation_date = g.app.get_timestamp()
# Insert the MeasureExcludedGeographicalArea
if transform_only is False:
sql = """
insert into measure_excluded_geographical_areas_oplog
(measure_sid, excluded_geographical_area, geographical_area_sid,
operation, operation_date, created_at, filename)
VALUES (%s, %s, %s, %s, %s, %s, %s)
"""
params = [
self.measure_sid, self.excluded_geographical_area,
self.geographical_area_sid, self.operation, operation_date,
operation_date, import_file
]
d = Database()
d.run_query(sql, params)
def test_05_cancel_order(self):
"""取消订单"""
unpaid_data = {"session": self.setUp(), "type": "await_pay", "pagination": {"count": 10, "page": 1}}
"获取未支付订单中的order_id"
order_ids=Unpaid_order_interface().get_order_ids(unpaid_data)
"随机获取一个order_id"
random_num=random.randint(0,len(order_ids)-1)
cancel_order_id=order_ids[random_num]
print(cancel_order_id)
"把随机获取的order_id传入cancel_data中"
cancel_data={"session":self.setUp(),"order_id":cancel_order_id}
"删除订单"
Cancel_order().cancel_order(cancel_data)
"""连接数据库"""
database = Database(host="ecshop.itsoso.cn", user="******", password="******", database="ecshop",
charset='utf8', port=3306)
sql = "select * from ecs_order_info where user_id=11402 and order_status=3"
order_id_db = database.readall(sql=sql)
print(order_id_db)
"""在数据库中获取取消订单的id"""
db_orderid = get_keywords(order_id_db, "order_id")
print(db_orderid)
"""断言"""
"""判断取消订单的id是否存在于数据库取消订单的id中"""
if order_id_db:
if cancel_order_id in db_orderid:
self.assertTrue(True)
else:
self.assertTrue(False)
else:
print("数据库中订单状态被取消的订单为零")
def request(self):
try:
with Database() as session:
with Browser() as browser_session:
url = self.url(Category.NEWSLETTER.value)
response = browser_session.get(url)
response.raise_for_status()
soup = bs4.BeautifulSoup(response.content, 'html.parser')
elements = soup.select('#content-left a')
LOGGER.info('{} newsletters have been found'.format(
len(elements)))
if len(elements) == 0:
LOGGER.info('Done, no more newsletters')
return
for element in reversed(elements):
href = element['href']
title = element.text
model, created = session.get_or_create(Newsletter,
url=href,
title=title)
if not created:
LOGGER.info(
f'Newsletter "{href}" ...skipped (duplicate)')
else:
LOGGER.info(f'{href} ...added')
except requests.exceptions.HTTPError as exc:
LOGGER.warning('Scraping {} {} ...skipping'.format(
url, exc.response.status_code))
raise exc
def main(self):
if self.brute is None:
self.brute = config.enable_brute_module
if self.verify is None:
self.verify = config.enable_verify_subdomain
rename_table = self.domain + '_last'
collect = Collect(self.domain, export=False)
collect.run()
if self.brute:
# 由于爆破会有大量dns解析请求 并发爆破可能会导致其他任务中的网络请求异常
brute = AIOBrute(self.domain, export=False)
brute.run()
db = Database()
db.copy_table(self.domain, self.domain + '_ori')
db.remove_invalid(self.domain)
db.deduplicate_subdomain(self.domain)
# 不验证子域的情况
if not self.verify:
# 数据库导出
self.valid = None
dbexport.export(self.domain,
valid=self.valid,
format=self.format,
show=self.show)
db.drop_table(rename_table)
db.rename_table(self.domain, rename_table)
return
# 开始验证子域工作
self.datas = db.get_data(self.domain).as_dict()
loop = asyncio.get_event_loop()
asyncio.set_event_loop(loop)
# 解析域名地址
task = resolve.bulk_query_a(self.datas)
self.datas = loop.run_until_complete(task)
# 保存解析结果
resolve_table = self.domain + '_res'
db.drop_table(resolve_table)
db.create_table(resolve_table)
db.save_db(resolve_table, self.datas, 'resolve')
# 请求域名地址
task = request.bulk_get_request(self.datas, self.port)
self.datas = loop.run_until_complete(task)
# 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
loop.run_until_complete(asyncio.sleep(0.25))
db.clear_table(self.domain)
db.save_db(self.domain, self.datas)
# 数据库导出
dbexport.export(self.domain,
valid=self.valid,
format=self.format,
show=self.show)
db.drop_table(rename_table)
db.rename_table(self.domain, rename_table)
db.close()
def __init__(self,
elem,
quota_order_number_origin_sid,
import_file,
transform_only=False):
Master.__init__(self, elem)
self.quota_order_number_origin_sid = quota_order_number_origin_sid
self.excluded_geographical_area_sid = Master.process_null(
elem.find("geographicalArea/sid"))
operation_date = g.app.get_timestamp()
if transform_only is False:
sql = """
insert into quota_order_number_origin_exclusions_oplog
(quota_order_number_origin_sid, excluded_geographical_area_sid,
operation, operation_date, created_at, filename)
VALUES (%s, %s, %s, %s, %s, %s)
"""
params = [
self.quota_order_number_origin_sid,
self.excluded_geographical_area_sid, self.operation,
operation_date, operation_date, import_file
]
d = Database()
d.run_query(sql, params)
def test_ifnotinput_serach(self):
"""
用例ID: 3
用例标题:验证用户不输入搜索条件,点击搜索后显示所有商品信息
预置条件:使用谷歌浏览器进入首页
执行步骤:
1.不输入搜索条件,点击搜索按钮
预期结果:
1.成功显示所有商品信息
:return:
"""
ugs = UserGoodsSearch(self.driver)
ugs.click_serach_button()
try:
count1 = ugs.count_current_goods()
ugs.next_goods_page()
count2 = ugs.count_current_goods()
ugs.next_goods_page()
count3 = ugs.count_current_goods()
ugs.next_goods_page()
coun4 = ugs.count_current_goods()
allgoods = count1 + count2 + count3 + coun4
except Exception as e:
self.assertTrue(False, msg=f'没有找到这个商品{e}')
else:
db = Database(database='ecshop', password='******')
sql = 'select count(*) from ec_goods where is_on_sale=1'
dabase_count = db.read_one(sql)['count(*)']
self.assertTrue(int(allgoods) == dabase_count, msg='商品没有全部显示')
def __init__(self, elem, measure_sid, import_file, transform_only=False):
Master.__init__(self, elem)
self.measure_sid = measure_sid
self.duty_expression_id = Master.process_null(elem.find("dutyExpression/dutyExpressionId"))
self.duty_amount = Master.process_null(elem.find("dutyAmount"))
self.monetary_unit_code = Master.process_null(elem.find("monetaryUnit/monetaryUnitCode"))
self.measurement_unit_code = Master.process_null(elem.find("measurementUnit/measurementUnitCode"))
self.measurement_unit_qualifier_code = Master.process_null(elem.find("measurementUnitQualifier/measurementUnitQualifierCode"))
operation_date = g.app.get_timestamp()
# Insert the MeasureComponent
if transform_only is False:
sql = """
insert into measure_components_oplog
(measure_sid, duty_expression_id, duty_amount,
monetary_unit_code, measurement_unit_code, measurement_unit_qualifier_code,
operation, operation_date, created_at, filename)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
"""
params = [
self.measure_sid,
self.duty_expression_id,
self.duty_amount,
self.monetary_unit_code,
self.measurement_unit_code,
self.measurement_unit_qualifier_code,
self.operation,
operation_date,
operation_date,
import_file
]
d = Database()
d.run_query(sql, params)
def __init__(self,
elem,
additional_code_type_id,
import_file,
transform_only=False):
Master.__init__(self, elem)
self.additional_code_type_id = additional_code_type_id
self.measure_type_id = Master.process_null(
elem.find("measureType/measureTypeId"))
self.validity_start_date = Master.process_date(
elem.find("validityStartDate"))
self.validity_end_date = Master.process_date(
elem.find("validityEndDate"))
operation_date = g.app.get_timestamp()
g.app.print_only(
"Running operation {0} on additional code type measure type {1}.".
format(self.operation, self.additional_code_type_id))
if transform_only is False:
sql = """
insert into additional_code_type_measure_types_oplog
(additional_code_type_id, measure_type_id,
validity_start_date, validity_end_date,
operation, operation_date, created_at, filename)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
"""
params = [
self.additional_code_type_id, self.measure_type_id,
self.validity_start_date, self.validity_end_date,
self.operation, operation_date, operation_date, import_file
]
d = Database()
d.run_query(sql, params)
def get_commodity_change_end(self):
sql = """
with cte as (
select distinct on (gn.goods_nomenclature_sid)
gn.goods_nomenclature_item_id, gn.producline_suffix, gnd.description,
gn.validity_start_date, gn.validity_end_date
from goods_nomenclatures gn, goods_nomenclature_descriptions gnd
where gn.goods_nomenclature_sid = gnd.goods_nomenclature_sid
and validity_end_date >= %s
and validity_end_date <= %s
order by gn.goods_nomenclature_sid
)
select * from cte order by goods_nomenclature_item_id;
"""
params = [
self.from_date,
self.to_date
]
d = Database()
rows = d.run_query(sql, params)
if len(rows) > 0:
for row in rows:
delta_commodity = DeltaCommodity(row, "Close")
self.delta_commodities.append(delta_commodity)
def find_subdomains(domain, data):
subdomains = set()
js_urls = set()
db = Database()
for infos in data:
jump_history = infos.get('history')
req_url = infos.get('url')
subdomains.update(find_in_history(domain, req_url, jump_history))
rsp_html = db.get_resp_by_url(domain, req_url)
if not rsp_html:
logger.log(
'DEBUG',
f'an abnormal response occurred in the request {req_url}')
continue
subdomains.update(find_in_resp(domain, req_url, rsp_html))
js_urls.update(find_js_urls(domain, req_url, rsp_html))
req_data = convert_to_dict(js_urls)
resp_data = request.bulk_request(domain, req_data, ret=True)
while not resp_data.empty():
_, resp = resp_data.get()
if not isinstance(resp, Response):
continue
text = utils.decode_resp_text(resp)
subdomains.update(find_in_resp(domain, resp.url, text))
return subdomains
def __init__(self,
elem,
goods_nomenclature_description_period_sid,
goods_nomenclature_sid,
goods_nomenclature_item_id,
productline_suffix,
import_file,
transform_only=False):
Master.__init__(self, elem)
self.goods_nomenclature_description_period_sid = goods_nomenclature_description_period_sid
self.goods_nomenclature_sid = goods_nomenclature_sid
self.goods_nomenclature_item_id = goods_nomenclature_item_id
self.productline_suffix = productline_suffix
self.description = Master.process_null(elem.find("description"))
self.language_id = Master.process_null(
elem.find("language/languageId"))
operation_date = g.app.get_timestamp()
if transform_only is False:
sql = """
insert into goods_nomenclature_descriptions_oplog
(goods_nomenclature_description_period_sid, goods_nomenclature_sid,
goods_nomenclature_item_id, productline_suffix, language_id, description,
operation, operation_date, created_at, filename)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
"""
params = [
self.goods_nomenclature_description_period_sid,
self.goods_nomenclature_sid, self.goods_nomenclature_item_id,
self.productline_suffix, self.language_id, self.description,
self.operation, operation_date, operation_date, import_file
]
d = Database()
d.run_query(sql, params)
def __init__(self,
elem,
certificate_description_period_sid,
certificate_type_code,
certificate_code,
import_file,
transform_only=False):
Master.__init__(self, elem)
self.certificate_description_period_sid = certificate_description_period_sid
self.certificate_type_code = certificate_type_code
self.certificate_code = certificate_code
self.description = Master.process_null(elem.find("description"))
self.language_id = Master.process_null(
elem.find("language/languageId"))
operation_date = g.app.get_timestamp()
g.app.print_only(
"Running operation {0} on certificate descrition {1}.".format(
self.operation, str(self.certificate_description_period_sid)))
if transform_only is False:
sql = """
insert into certificate_descriptions_oplog
(certificate_description_period_sid, certificate_type_code, certificate_code, language_id, description,
operation, operation_date, created_at, filename)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
"""
params = [
self.certificate_description_period_sid,
self.certificate_type_code, self.certificate_code,
self.language_id, self.description, self.operation,
operation_date, operation_date, import_file
]
d = Database()
d.run_query(sql, params)
def export(table, db=None, valid=None, dpath=None, format='csv', show=False):
"""
OneForAll数据库导出模块
Example:
python3 dbexport.py --table name --format csv --dir= ./result.csv
python3 dbexport.py --db result.db --table name --show False
Note:
参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
参数dpath为None默认使用OneForAll结果目录
:param str table: 要导出的表
:param str db: 要导出的数据库路径(默认为results/result.sqlite3)
:param int valid: 导出子域的有效性(默认None)
:param str format: 导出格式(默认csv)
:param str dpath: 导出目录(默认None)
:param bool show: 终端显示导出数据(默认False)
"""
format = utils.check_format(format)
dpath = utils.check_dpath(dpath)
database = Database(db)
rows = database.export_data(table, valid) # 意外情况导出全部子域
if show:
print(rows.dataset)
if format == 'txt':
data = str(rows.dataset)
else:
data = rows.export(format)
database.close()
fpath = dpath.joinpath(f'{table}_subdomain.{format}')
utils.save_data(fpath, data)
def export(table, db=None, valid=None, path=None, format='csv', show=False):
"""
OneForAll数据库导出模块
Example:
python3 dbexport.py --table name --format csv --dir= ./result.csv
python3 dbexport.py --db result.db --table name --show False
Note:
参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
参数path默认None使用OneForAll结果目录生成路径
:param str table: 要导出的表
:param str db: 要导出的数据库路径(默认为results/result.sqlite3)
:param int valid: 导出子域的有效性(默认None)
:param str format: 导出文件格式(默认csv)
:param str path: 导出文件路径(默认None)
:param bool show: 终端显示导出数据(默认False)
"""
database = Database(db)
rows = database.export_data(table, valid)
format = utils.check_format(format, len(rows))
path = utils.check_path(path, table, format)
if show:
print(rows.dataset)
if format == 'txt':
data = str(rows.dataset)
else:
data = rows.export(format)
database.close()
utils.save_data(path, data)
def export(table, db=None, valid=False, limit=None, path=None, format='csv', show=False):
"""
OneForAll数据库导出模块
Example:
python3 dbexport.py --table name --format csv --dir= ./result.csv
python3 dbexport.py --db result.db --table name --show False
Note:
参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
参数path默认None使用OneForAll结果目录自动生成路径
:param str table: 要导出的表
:param str db: 要导出的数据库路径(默认为results/result.sqlite3)
:param bool valid: 只导出有效的子域结果(默认False)
:param str limit: 导出限制条件(默认None)
:param str format: 导出文件格式(默认csv)
:param str path: 导出文件路径(默认None)
:param bool show: 终端显示导出数据(默认False)
"""
database = Database(db)
rows = database.export_data(table, valid, limit)
format = utils.check_format(format, len(rows))
path = utils.check_path(path, table, format)
if show:
print(rows.dataset)
data = rows.export(format)
database.close()
utils.save_data(path, data)
logger.log('INFOR', f'{table}主域的子域结果 {path}')
data_dict = rows.as_dict()
return data_dict
def test_06_pay_order(self):
"""支付订单"""
"先查看未支付订单"
unpaid_data = {"session": self.setUp(), "type": "await_pay", "pagination": {"count": 10, "page": 1}}
"获取未支付订单中的order_id"
order_ids = Unpaid_order_interface().get_order_ids(unpaid_data)
"随机获取一个order_id"
random_num = random.randint(0, len(order_ids)-1)
print(random_num)
pay_order_id = order_ids[random_num]
print(pay_order_id)
"把随机获取的order_id传入pay_data中"
pay_data={"session":self.setUp(),"order_id":pay_order_id}
"支付订单"
Pay_order().pay_order(pay_data)
"""连接数据库"""
database = Database(host="ecshop.itsoso.cn", user="******", password="******", database="ecshop",
charset='utf8', port=3306)
sql = "select * from ecs_order_info where user_id=11402 and pay_status=2"
order_id_db = database.readall(sql=sql)
print(order_id_db)
db_orderid = get_keywords(order_id_db, "order_id")
print(db_orderid)
"""断言"""
if pay_order_id in db_orderid:
self.assertTrue(True)
else:
self.assertTrue(False)
def __init__(self,
elem,
footnote_description_period_sid,
footnote_type_id,
footnote_id,
import_file,
transform_only=False):
Master.__init__(self, elem)
self.footnote_description_period_sid = footnote_description_period_sid
self.footnote_type_id = footnote_type_id
self.footnote_id = footnote_id
self.description = Master.process_null(elem.find("description"))
self.language_id = Master.process_null(
elem.find("language/languageId"))
operation_date = g.app.get_timestamp()
if transform_only is False:
sql = """
insert into footnote_descriptions_oplog
(footnote_description_period_sid, footnote_type_id, footnote_id, language_id, description,
operation, operation_date, created_at, filename)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
"""
params = [
self.footnote_description_period_sid, self.footnote_type_id,
self.footnote_id, self.language_id, self.description,
self.operation, operation_date, operation_date, import_file
]
d = Database()
d.run_query(sql, params)
def deal_db(self):
"""
Process the data when the collection task is completed
"""
db = Database()
db.deal_table(self.domain, self.origin_table)
db.close()
def request(self):
with Database() as session:
with Browser() as browser_session:
url = self.url(Category.INVOICE.value)
loop = True
while loop:
try:
response = browser_session.get(url)
response.raise_for_status()
soup = bs4.BeautifulSoup(response.content,
'html.parser')
elements = soup.select(
'#dokumenty table.tabulka tr:not(.hlavicka)')
if len(elements) == 0:
LOGGER.info('Done, no more invoices')
return
for element in elements:
published, _, title, _, _, _, document = element.findChildren(
'td')
link = document.findChild('a').attrs.get('href')
size_in_mb = re.search(r'([0-9\.]+)',
document.text).groups()[0]
is_pdf = re.search(r'\.pdf$', url)
if is_pdf:
model, created = session.get_or_create(
Invoice,
published=datetime.date.fromisoformat(
published.text),
title=title.text,
url=link,
size_in_mb=size_in_mb)
if not created:
LOGGER.info(
f'Invoice {model.url} ...skipped (duplicate)'
)
loop = False
else:
LOGGER.info(f'{model.url} ...added')
else:
LOGGER.warning(
f'Invoice {model.url} ...skipped (not PDF)'
)
next_url = soup.select_one(
'#dokumenty table:first-of-type [align="right"] a:nth-last-child(2)'
).attrs.get('href')
next_url = urljoin(self.base_url(response.url),
next_url)
# FIXME: first page can be w/o the page number
if next_url == url:
return
url = next_url
except requests.exceptions.HTTPError as exc:
LOGGER.warning('Scraping {} {} ...skipping'.format(
url, exc.response.status_code))
raise exc
